Context Edit:20170530 Googlenews vectors are good because they can be memory mapped. Spacy doesn't have word distance mover and is still buggy as hell. Gensim has wdm, and can do it from mmap'd word2vec files. All you linux users... omg don't get me started. make myass.file. Have to trim down this shit for the layman trying to get a leg up on this comp. First step... get unique words and then get the unique vectors from the google news linux crap bzip2. Content -- \Kaggle\Quora_20170422\gen_sim_crap\Get_Unique_Words_in_data_v2.py TEST_FILE = os.path.join(BASE_DIR, r'testcsv', r'test.csv') 94%|█████████▍| 2345806/2500000 [00:54<00:03, 42961.02it/s] 55.024147272109985 -- len(unique_words) = 124848 Acknowledgements We wouldn't be here without the help of others. If you owe any attributions or thanks, include them here along with any citations of past research. Inspiration import time, pickle, os from tqdm import tqdm import string OUT_FILE = r'\Kaggle\Quora_20170422\FeatureEngineering\ListofUniqueTest.pkl' BASE_DIR = r'\Documents\Kaggle\Quora_20170422' #TRAIN_FILE = os.path.join(BASE_DIR, r'traincsv', r'train.csv') TEST_FILE = os.path.join(BASE_DIR, r'testcsv', r'test.csv') def get_words(f, c2r): with tqdm(total=2500000) as pbar: for line in f: #b_string = line.replace(',', ' ') c_string = line.translate(str.maketrans({key: ' ' for key in c2r})) pbar.update() for word in c_string.split(): yield word start_time = time.time() chars_to_replace = string.punctuation + string.digits with open(TEST_FILE, encoding="utf8") as infile: unique_words = sorted(set(get_words(infile, chars_to_replace))) pickle.dump(unique_words, open(OUT_FILE, 'wb'), -1) elapsed_time = time.time() - start_time print(elapsed_time)