--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/classify_tweets.py	Fri Jan 17 15:45:26 2025 +0000
@@ -0,0 +1,130 @@
+from collections import defaultdict, Counter
+from typing import Tuple, List, Any, Set, Dict, Callable
+
+import numpy as np  # for np.mean() and np.std()
+import nltk, sys, inspect
+import nltk.corpus.util
+from nltk import MaxentClassifier
+from nltk.corpus import brown, ppattach  # import corpora
+
+# Import LgramModel
+from nltk_model import *
+
+# Import the Twitter corpus
+from twitter.twitter import *
+
+twitter_file_ids = "20100128.txt"
+assert twitter_file_ids in xtwc.fileids()
+
+
+def train_LM(corpus: nltk.corpus.CorpusReader) -> LgramModel:
+    """
+    Build a bigram letter language model using LgramModel
+    based on the lower-cased all-alpha subset of the entire corpus
+
+    :param corpus: An NLTK corpus
+
+    :return: A padded letter bigram model based on nltk.model.NgramModel
+    """
+
+    # subset the corpus to only include all-alpha tokens,
+    # converted to lower-case (_after_ the all-alpha check)
+    corpus_tokens = [word.lower() for word in corpus.words() if word.isalpha()]
+
+    # Return the tokens and a smoothed (using the default estimator)
+    #   padded bigram letter language model
+    return LgramModel(2, corpus_tokens, True, True)
+
+
+lm = train_LM(brown)
+
+def rtc(x):
+    counter = 0
+    last_char = None
+    o = []
+    for i, c in enumerate(x):
+        if c == last_char:
+            counter += 1
+        else:
+            counter = 0
+        if counter < 2:
+            o.append(c)
+        last_char = c
+    return "".join(o)
+
+
+import re
+def scoring_f(bigram_model: LgramModel, tweet: List[str]) -> bool:
+    """
+    Classify if the given tweet is written in English or not.
+
+    :param bigram_model: the bigram letter model trained on the Brown corpus
+    :param tweet: the tweet
+    :return: True if the tweet is classified as English, False otherwise
+    """
+    blacklist = {"rt", "smh", "hbu", "idk", "afaik","imho", "irl"}
+    blacklist_regex = ["lo+l+", "a+w+", "ya+y+", "o+m+g+","lma+o+"]
+    prepro = []
+    for token in tweet:
+        token = rtc(token)
+        if token in blacklist:
+            continue
+        elif any(re.fullmatch(regex, token) for regex in blacklist_regex):
+            continue
+        else:
+            prepro.append(token)
+    tweet = prepro
+    return sum([bigram_model.entropy(token, perItem=True) for token in tweet]) / len(tweet)
+
+def boring_scoring(bigram_model, tweet):
+    return sum([bigram_model.entropy(token, perItem=True) for token in tweet]) / len(tweet)
+
+def is_English(tweet,thresh=None):
+    #return boring_scoring(lm, tweet) <= 3.85
+    #return boring_scoring(lm, tweet) <= 3.31 # lower threshold needed to get dev data right
+    #return boring_scoring(lm, tweet) < 6.182 # upper threshold needed to get dev data right
+    return scoring_f(lm, tweet) < (3.85 if thresh == None else thresh) # well-tuned threshold
+
+def get_tweets(fn: str,n):
+    """
+    :rtype list(tuple(list(str), bool))
+    :return: a list of tuples (tweet, a) where tweet is a tweet preprocessed by us,
+    and a is True, if the tweet is in English, and False otherwise.
+    """
+    f=open(fn)
+    i=0
+    for l in f:
+        if n>0 and i==n:
+            break
+        yield l.split()
+        i+=1
+    f.close()
+
+def eval(n1=0,n2=0,thresh=None):
+    fp=tp=fn=tn=0
+    np=nn=0
+    for tweet in get_tweets("twitter/etweets.txt",n1):
+        np+=1
+        if is_English(tweet,thresh):
+            tp+=1
+        else:
+            fp+=1
+    for tweet in get_tweets("twitter/netweets.txt",n2):
+        nn+=1
+        if is_English(tweet,thresh):
+            fn+=1
+        else:
+            tn+=1
+    print("Testing on %s/%s tweets, threshhold %s"%('all' if n1==0 else n1,
+                                                    'all' if n2==0 else n2,
+                                              'default' if thresh==None else thresh))
+    print("%6s %6s %6s %6s"%('','right','wrong','total'))
+    print("%6s %6d %6d %6d"%('pos',tp,fp,np))
+    print("%6s %6d %6d %6d"%('neg',tn,fn,nn))
+    print("%6s %6d %6d %6d"%('tot',tp+tn,fp+fn,np+nn))
+    print("Accuracy: %g"%(float(tp+tn)/float(np+nn)))
+
+eval(100)
+
+
+