notebooks/gen_face_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "import math\n",
    "from collections import defaultdict\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_word(word_length):\n",
    "    word_len = np.random.randint(*word_length)\n",
    "    return ''.join(random.sample(string.ascii_lowercase, word_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_size = 6\n",
    "num_topics = 20\n",
    "\n",
    "topics = [\n",
    "    [gen_word((2,6)) for _ in range(topic_size)] for _ in range(num_topics)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['srpq', 'mfvt', 'oicel', 'so', 'ov', 'rmg'],\n",
       " ['hs', 'cjhrt', 'gr', 'nbtz', 'vnbf', 'pjn'],\n",
       " ['migw', 'gx', 'tyhfl', 'vfim', 'qtd', 'es'],\n",
       " ['gfsrc', 'dvr', 'xyn', 'nzr', 'ks', 'wnk'],\n",
       " ['khvd', 'sd', 'xh', 'xurpb', 'tbzh', 'xpfh'],\n",
       " ['akqw', 'ybov', 'bacj', 'vawin', 'lirjt', 'yigk'],\n",
       " ['kmbd', 'lvhm', 'wzr', 'em', 'typm', 'cxj'],\n",
       " ['abzxw', 'qeg', 'zsl', 'ra', 'aowf', 'rpb'],\n",
       " ['arg', 'em', 'kic', 'by', 'gin', 'ovq'],\n",
       " ['fuwyi', 'av', 'hmb', 'eurjf', 'bur', 'ad'],\n",
       " ['kwejo', 'ku', 'pgns', 'ztu', 'snm', 'edpmj'],\n",
       " ['uoyxf', 'hgr', 'xg', 'swb', 'emns', 'hd'],\n",
       " ['bvl', 'eh', 'kb', 'bf', 'rn', 'rnlg'],\n",
       " ['dwbn', 'atds', 'dklyp', 'pmj', 'arsd', 'tcvma'],\n",
       " ['ufp', 'omh', 'ex', 'twygo', 'rh', 'vabxq'],\n",
       " ['cenzb', 'ak', 'lps', 'hkgm', 'npkji', 'ozv'],\n",
       " ['ozhkl', 'ib', 'zxtrg', 'qkpg', 'yw', 'rp'],\n",
       " ['nr', 'fdbv', 'djt', 'jcfvh', 'lumt', 'bs'],\n",
       " ['qs', 'ukhbn', 'zxy', 'abgc', 'omlt', 'yajgk'],\n",
       " ['mv', 'kytnc', 'bjfs', 'cfs', 'dqmec', 'opkbh']]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_sentence(m_topics):\n",
    "    sent = []\n",
    "    prefix_topics = 1\n",
    "    suffix_topics = 1\n",
    "    \n",
    "    for i in range(prefix_topics):\n",
    "        sent += random.sample(topics[random.choice(range(num_topics))], random.randint(0,2))\n",
    "    \n",
    "    mention = random.choice(random.choice(topics))\n",
    "    \n",
    "    sent += random.sample(topics[random.choice(m_topics)], random.randint(0,2)) + [\"\\t\", mention, \"\\t\"] + random.sample(topics[random.choice(m_topics)], random.randint(0,2))\n",
    "    \n",
    "    for i in range(suffix_topics):\n",
    "        sent += random.sample(topics[random.choice(range(num_topics))], random.randint(0,2))\n",
    "      \n",
    "    return sent\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([4, 12], ['khvd', '\\t', 'oicel', '\\t', 'swb'])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_topics = random.choices(range(num_topics), k=2)\n",
    "m_topics, generate_sentence(m_topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_to_file(file_path, count=1000):\n",
    "    with open(file_path, 'w') as out:\n",
    "        for i in range(count):\n",
    "            m_topics = random.choices(range(num_topics), k=2)\n",
    "            topics_str = \"_\".join(map(str, m_topics))\n",
    "            for j in range(random.randint(2, 4)):\n",
    "                sent = generate_sentence(m_topics)\n",
    "                out.write(topics_str + \"\\t\" + ' '.join(sent) + '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_to_file('../data/fake_data_train.tsv', count=10000)\n",
    "generate_to_file('../data/fake_data_valid.tsv', count=500)          \n",
    "generate_to_file('../data/fake_data_test.tsv', count=500) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/fake_ft_data.tsv', 'w') as out:\n",
    "    for i in range(1000):\n",
    "        m_topics = random.choices(range(num_topics), k=2)\n",
    "        topics_str = \"_\".join(map(str, m_topics))\n",
    "        for j in range(random.randint(2, 4)):\n",
    "            sent = generate_sentence(m_topics)\n",
    "            out.write(' '.join(sent) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install git+https://github.com/facebookresearch/fastText.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fastText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = fastText.train_unsupervised(\n",
    "    input='../data/fake_ft_data.tsv', minCount=0, bucket=1000, dim=30\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.get_words())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_model(\"../data/fake_ft_model.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import FastText\n",
    "from gensim.models.utils_any2vec import ft_ngram_hashes\n",
    "from gensim.models.utils_any2vec import compute_ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `load_fasttext_format` (use load_facebook_vectors (to use pretrained embeddings) or load_facebook_model (to continue training with the loaded full model, more RAM) instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "ft = FastText.load_fasttext_format(\"../data/fake_ft_model.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ft.save('../data/gensim_fake_ft.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "ft2 = FastText.load('../data/gensim_fake_ft.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `min_n` (Attribute will be removed in 4.0.0, use wv.min_n instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ft2.min_n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "topics_mapping = {}\n",
    "for idx, _ in enumerate(topics):\n",
    "    for idx2, _ in enumerate(topics):\n",
    "        topics_mapping[f\"{idx}_{idx2}\"] = [str(idx), str(idx2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/category_mapping.json', 'w') as out:\n",
    "    json.dump(topics_mapping, out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "creating file '../data/fake_data_train.tsv_aa'\n",
      "creating file '../data/fake_data_train.tsv_ab'\n",
      "creating file '../data/fake_data_train.tsv_ac'\n",
      "creating file '../data/fake_data_train.tsv_ad'\n",
      "creating file '../data/fake_data_train.tsv_ae'\n",
      "creating file '../data/fake_data_train.tsv_af'\n",
      "creating file '../data/fake_data_train.tsv_ag'\n",
      "creating file '../data/fake_data_train.tsv_ah'\n",
      "creating file '../data/fake_data_train.tsv_ai'\n",
      "creating file '../data/fake_data_train.tsv_aj'\n"
     ]
    }
   ],
   "source": [
    "!split -n l/10 --verbose ../data/fake_data_train.tsv ../data/fake_data_train.tsv_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting google-cloud-storage\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/aa/048f5b3950f78c9e6afdb05e3667abb7a7ca4463bfde002257acd1874c3f/google_cloud_storage-1.15.0-py2.py3-none-any.whl (64kB)\n",
      "\u001b[K    100% |████████████████████████████████| 71kB 653kB/s \n",
      "\u001b[?25hCollecting google-cloud-core<0.30dev,>=0.29.0 (from google-cloud-storage)\n",
      "  Using cached https://files.pythonhosted.org/packages/0c/f2/3c225e7a69cb27d283b68bff867722bd066bc1858611180197f711815ea5/google_cloud_core-0.29.1-py2.py3-none-any.whl\n",
      "Collecting google-api-core<2.0.0dev,>=1.6.0 (from google-cloud-storage)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bf/e4/b22222bb714947eb459dc91ebf95131812126a0b29d62e444be3f76dad64/google_api_core-1.9.0-py2.py3-none-any.whl (65kB)\n",
      "\u001b[K    100% |████████████████████████████████| 71kB 10.3MB/s \n",
      "\u001b[?25hCollecting google-resumable-media>=0.3.1 (from google-cloud-storage)\n",
      "  Using cached https://files.pythonhosted.org/packages/e2/5d/4bc5c28c252a62efe69ed1a1561da92bd5af8eca0cdcdf8e60354fae9b29/google_resumable_media-0.3.2-py2.py3-none-any.whl\n",
      "Requirement already satisfied: protobuf>=3.4.0 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (3.6.1)\n",
      "Collecting google-auth<2.0dev,>=0.4.0 (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage)\n",
      "  Using cached https://files.pythonhosted.org/packages/c5/9b/ed0516cc1f7609fb0217e3057ff4f0f9f3e3ce79a369c6af4a6c5ca25664/google_auth-1.6.3-py2.py3-none-any.whl\n",
      "Collecting googleapis-common-protos!=1.5.4,<2.0dev,>=1.5.3 (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage)\n",
      "  Downloading https://files.pythonhosted.org/packages/76/a6/903a1c1c1fd676660174a9be4ec777c9b547392f640b16a326182b8fe484/googleapis-common-protos-1.5.9.tar.gz\n",
      "Requirement already satisfied: pytz in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (2017.3)\n",
      "Requirement already satisfied: six>=1.10.0 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (1.12.0)\n",
      "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (2.21.0)\n",
      "Requirement already satisfied: setuptools>=34.0.0 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (40.8.0)\n",
      "Collecting cachetools>=2.0.0 (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage)\n",
      "  Using cached https://files.pythonhosted.org/packages/39/2b/d87fc2369242bd743883232c463f28205902b8579cb68dcf5b11eee1652f/cachetools-3.1.0-py2.py3-none-any.whl\n",
      "Collecting pyasn1-modules>=0.2.1 (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage)\n",
      "  Using cached https://files.pythonhosted.org/packages/da/98/8ddd9fa4d84065926832bcf2255a2b69f1d03330aa4d1c49cc7317ac888e/pyasn1_modules-0.2.4-py2.py3-none-any.whl\n",
      "Requirement already satisfied: rsa>=3.1.4 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (3.4.2)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (1.24.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (2018.11.29)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.1 in /home/generall/sources/Sci/ml/allen_evn/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=0.4.0->google-api-core<2.0.0dev,>=1.6.0->google-cloud-storage) (0.4.5)\n",
      "Building wheels for collected packages: googleapis-common-protos\n",
      "  Building wheel for googleapis-common-protos (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /home/generall/.cache/pip/wheels/b8/16/23/0a02129b4da233fbc8d08d39317367f6b9708d62f3eb641649\n",
      "Successfully built googleapis-common-protos\n",
      "Installing collected packages: cachetools, pyasn1-modules, google-auth, googleapis-common-protos, google-api-core, google-cloud-core, google-resumable-media, google-cloud-storage\n",
      "Successfully installed cachetools-3.1.0 google-api-core-1.9.0 google-auth-1.6.3 google-cloud-core-0.29.1 google-cloud-storage-1.15.0 google-resumable-media-0.3.2 googleapis-common-protos-1.5.9 pyasn1-modules-0.2.4\n",
      "\u001b[33mYou are using pip version 19.0.2, however version 19.0.3 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install google-cloud-storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}