blunt-octopus/transformers-devops
0
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 1,6 "metadata": {7 "cellId": "nwmnsa077j0li6frzyvoc"8 },9 "outputs": [],10 "source": [11 "import torch\n",12 "import numpy as np\n",13 "import json\n",14 "from tqdm.auto import tqdm"15 ]16 },17 {18 "cell_type": "markdown",19 "metadata": {20 "cellId": "dfsqzm9i1rg0djpmrynj4"21 },22 "source": [23 "Для начала распарсим датасет \"по умолчанию\"."24 ]25 },26 {27 "cell_type": "code",28 "execution_count": 2,29 "metadata": {30 "cellId": "g133nsvyrtio3fpwpn54m"31 },32 "outputs": [],33 "source": [34 "import json\n",35 "file = open('arxivData.json')\n",36 "data = json.load(file)"37 ]38 },39 {40 "cell_type": "code",41 "execution_count": 3,42 "metadata": {43 "cellId": "04b7jpapxwa243bwo5a9fic"44 },45 "outputs": [],46 "source": [47 "def trl(container):\n",48 " return tqdm(range(len(container)))\n",49 "\n",50 "def prepared(string):\n",51 " string = string.replace(\"'\", '\"')\n",52 " string = string.replace('None', 'null')\n",53 " return string"54 ]55 },56 {57 "cell_type": "code",58 "execution_count": 322,59 "metadata": {60 "cellId": "5pr61iqulb4yxea8bjlhw"61 },62 "outputs": [63 {64 "data": {65 "application/vnd.jupyter.widget-view+json": {66 "model_id": "78535363d49b48889ed5beaf19af863c",67 "version_major": 2,68 "version_minor": 069 },70 "text/plain": [71 "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=41000.0), HTML(value='')))"72 ]73 },74 "metadata": {},75 "output_type": "display_data"76 },77 {78 "data": {79 "application/vnd.jupyter.widget-view+json": {80 "model_id": "dc1c7b04e9634db295cde22088934053",81 "version_major": 2,82 "version_minor": 083 },84 "text/plain": [85 "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=41000.0), HTML(value='')))"86 ]87 },88 "metadata": {},89 "output_type": "display_data"90 },91 {92 "data": {93 "application/vnd.jupyter.widget-view+json": {94 "model_id": "039e3d572c70452ca0710e6c61d10b80",95 "version_major": 2,96 "version_minor": 097 },98 "text/plain": [99 "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=41000.0), HTML(value='')))"100 ]101 },102 "metadata": {},103 "output_type": "display_data"104 },105 {106 "name": "stdout",107 "output_type": "stream",108 "text": [109 "\n",110 "\n",111 "\n"112 ]113 }114 ],115 "source": [116 "titles = [data[i]['title'] for i in trl(data)]\n",117 "summaries = [data[i]['summary'] for i in trl(data)]\n",118 "tags = [[json.loads(prepared(data[i]['tag']))[j]['term'] for j in range(len(json.loads(prepared(data[i]['tag']))))] for i in trl(data)]"119 ]120 },121 {122 "cell_type": "markdown",123 "metadata": {124 "cellId": "i36xlscxkm9t8ab4hye7c"125 },126 "source": [127 "Теперь соберем данные о тегах со страницы арксива."128 ]129 },130 {131 "cell_type": "code",132 "execution_count": 4,133 "metadata": {134 "cellId": "h0nt7pj6blgyodi07nt3e"135 },136 "outputs": [],137 "source": [138 "import requests\n",139 "page = requests.get(\"https://arxiv.org/category_taxonomy\")"140 ]141 },142 {143 "cell_type": "code",144 "execution_count": 5,145 "metadata": {146 "cellId": "eqd8ffof4m97mqffttt"147 },148 "outputs": [149 {150 "name": "stderr",151 "output_type": "stream",152 "text": [153 "/kernel/lib/python3.8/site-packages/ml_kernel/kernel.py:872: UserWarning: The following variables cannot be serialized: soup\n",154 " warnings.warn(message)\n"155 ]156 }157 ],158 "source": [159 "from bs4 import BeautifulSoup\n",160 "soup = BeautifulSoup(page.content, 'html.parser')"161 ]162 },163 {164 "cell_type": "code",165 "execution_count": 6,166 "metadata": {167 "cellId": "7lwv6ajhf8ewgolxzof1bm"168 },169 "outputs": [170 {171 "name": "stderr",172 "output_type": "stream",173 "text": [174 "/kernel/lib/python3.8/site-packages/ml_kernel/kernel.py:872: UserWarning: The following variables cannot be serialized: h4_cases, soup\n",175 " warnings.warn(message)\n"176 ]177 }178 ],179 "source": [180 "h4_cases = soup.find_all('h4')"181 ]182 },183 {184 "cell_type": "code",185 "execution_count": 7,186 "metadata": {187 "cellId": "ybnwppuruulh7ljycj7rh"188 },189 "outputs": [190 {191 "name": "stderr",192 "output_type": "stream",193 "text": [194 "/kernel/lib/python3.8/site-packages/ml_kernel/kernel.py:872: UserWarning: The following variables cannot be serialized: h4_cases, html_tag, soup, tags_tags\n",195 " warnings.warn(message)\n"196 ]197 }198 ],199 "source": [200 "tags_tags = h4_cases[1:]\n",201 "possible_tags = set()\n",202 "tag_to_id = {}\n",203 "id_to_tag = {}\n",204 "id_to_description = {}\n",205 "for i, html_tag in enumerate(tags_tags):\n",206 " idx = html_tag.get_text().find(' ')\n",207 " tag = html_tag.get_text()[:idx]\n",208 " description = html_tag.get_text()[idx+2:-1]\n",209 " possible_tags.add(tag)\n",210 " tag_to_id[tag] = i\n",211 " id_to_tag[i] = tag\n",212 " id_to_description[i] = description"213 ]214 },215 {216 "cell_type": "code",217 "execution_count": 8,218 "metadata": {219 "cellId": "pplhqb4oesl6pq0w2pr3fe"220 },221 "outputs": [],222 "source": [223 "num_tags = len(possible_tags)"224 ]225 },226 {227 "cell_type": "code",228 "execution_count": 328,229 "metadata": {230 "cellId": "vzlkypl5cyl3nagb85pphc"231 },232 "outputs": [],233 "source": [234 "with open('tags.txt', 'w') as file:\n",235 " for i in range(num_tags):\n",236 " file.write(str(id_to_tag[i]) + ' ' + str(id_to_description[i]) + '\\n')"237 ]238 },239 {240 "cell_type": "markdown",241 "metadata": {242 "cellId": "vsiuhkjhmqkygl63n2pk9"243 },244 "source": [245 "И переведем теги в индексы."246 ]247 },248 {249 "cell_type": "code",250 "execution_count": 329,251 "metadata": {252 "cellId": "m26ffr9sr4rc00y2kupo8q"253 },254 "outputs": [],255 "source": [256 "for i, line in enumerate(tags):\n",257 " line = list(set(line) & possible_tags)\n",258 " int_line = []\n",259 " for tag in line:\n",260 " int_line.append(tag_to_id[tag])\n",261 " tags[i] = int_line"262 ]263 },264 {265 "cell_type": "markdown",266 "metadata": {267 "cellId": "jgvhsk815g9k8wu66ftx"268 },269 "source": [270 "Разобъем данные на трейн и тест..."271 ]272 },273 {274 "cell_type": "code",275 "execution_count": 330,276 "metadata": {277 "cellId": "icw5n8u2se021wsdwlwio2"278 },279 "outputs": [],280 "source": [281 "from sklearn.model_selection import train_test_split\n",282 "\n",283 "titles_train, titles_test, summaries_train, summaries_test, tags_train, tags_test = train_test_split(titles, summaries, tags, test_size=0.1, random_state=42)"284 ]285 },286 {287 "cell_type": "markdown",288 "metadata": {289 "cellId": "rs6f6fyz5nn8x8ycz12m"290 },291 "source": [292 "И создадим датасеты и дадалоадеры."293 ]294 },295 {296 "cell_type": "code",297 "execution_count": 9,298 "metadata": {299 "cellId": "9mji2qz8u8q79ays6y1j6"300 },301 "outputs": [302 {303 "data": {304 "application/vnd.jupyter.widget-view+json": {305 "model_id": "d7c008df0c254cefbd8d2402ae5020ae",306 "version_major": 2,307 "version_minor": 0308 },309 "text/plain": [310 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=28.0), HTML(value='')))"311 ]312 },313 "metadata": {},314 "output_type": "display_data"315 },316 {317 "data": {318 "application/vnd.jupyter.widget-view+json": {319 "model_id": "b1e94a7ab27e40a9967f12f7011076c7",320 "version_major": 2,321 "version_minor": 0322 },323 "text/plain": [324 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=483.0), HTML(value='')))"325 ]326 },327 "metadata": {},328 "output_type": "display_data"329 },330 {331 "data": {332 "application/vnd.jupyter.widget-view+json": {333 "model_id": "c676bcfd5d8a49e9bb8b5c35ca64cd33",334 "version_major": 2,335 "version_minor": 0336 },337 "text/plain": [338 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=231508.0), HTML(value='')))"339 ]340 },341 "metadata": {},342 "output_type": "display_data"343 },344 {345 "data": {346 "application/vnd.jupyter.widget-view+json": {347 "model_id": "dc4baeeb13f04f5da7bc57094aa0828f",348 "version_major": 2,349 "version_minor": 0350 },351 "text/plain": [352 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=466062.0), HTML(value='')))"353 ]354 },355 "metadata": {},356 "output_type": "display_data"357 },358 {359 "name": "stdout",360 "output_type": "stream",361 "text": [362 "\n",363 "\n",364 "\n",365 "\n"366 ]367 }368 ],369 "source": [370 "#!g1.1\n",371 "from transformers import AutoTokenizer, AutoModel, pipeline, DistilBertForSequenceClassification\n",372 "\n",373 "base_model_name = 'distilbert-base-uncased'\n",374 "base_tokenizer = AutoTokenizer.from_pretrained(base_model_name)\n",375 "#base_model = AutoModel.from_pretrained(base_model_name).to('cuda')"376 ]377 },378 {379 "cell_type": "code",380 "execution_count": 9,381 "metadata": {382 "cellId": "1cumn09ceki8g3916s4e4"383 },384 "outputs": [385 {386 "data": {387 "text/plain": [388 "('./tokenizer/tokenizer_config.json',\n",389 " './tokenizer/special_tokens_map.json',\n",390 " './tokenizer/vocab.txt',\n",391 " './tokenizer/added_tokens.json',\n",392 " './tokenizer/tokenizer.json')"393 ]394 },395 "execution_count": 9,396 "metadata": {},397 "output_type": "execute_result"398 }399 ],400 "source": [401 "#!g1.1\n",402 "base_tokenizer.save_pretrained('./tokenizer')"403 ]404 },405 {406 "cell_type": "code",407 "execution_count": 10,408 "metadata": {409 "cellId": "3rka3rozlhe1pwg0ewqvfs"410 },411 "outputs": [],412 "source": [413 "def multi_hot(indexes, num_classes):\n",414 " return torch.zeros((num_classes,)).scatter_(0, torch.tensor(indexes), torch.ones((num_classes,)))"415 ]416 },417 {418 "cell_type": "code",419 "execution_count": 11,420 "metadata": {421 "cellId": "ieptlunauzkup3dj55hvnc"422 },423 "outputs": [],424 "source": [425 "from torch.utils.data import Dataset, DataLoader\n",426 "\n",427 "class ArXivDataset(Dataset):\n",428 " def __init__(self, titles, abstracts, tags, possible_tags, tokenizer):\n",429 " super(ArXivDataset, self).__init__()\n",430 " self. titles = titles\n",431 " self.abstracts = abstracts\n",432 " self.tags = tags\n",433 " self.possible_tags = possible_tags\n",434 " self.num_classes = len(self.possible_tags)\n",435 " self.tokenizer = tokenizer\n",436 " \n",437 " self.inputs = ['Title: ' + self.titles[idx] + ' Abstract: ' + self.abstracts[idx] for idx in range(len(titles))]\n",438 " self.inputs = self.tokenizer(self.inputs, truncation=True, padding='max_length', max_length=512, return_tensors='pt')['input_ids']\n",439 " \n",440 " def __len__(self):\n",441 " return len(self.titles)\n",442 " \n",443 " def __getitem__(self, idx):\n",444 " return (self.inputs[idx], multi_hot(self.tags[idx], self.num_classes) / len(self.tags[idx]))"445 ]446 },447 {448 "cell_type": "code",449 "execution_count": 334,450 "metadata": {451 "cellId": "vapsctsu94mjwcjydp4al"452 },453 "outputs": [],454 "source": [455 "#!g1.1\n",456 "base_train_ds = ArXivDataset(titles_train, summaries_train, tags_train, possible_tags, base_tokenizer)\n",457 "base_test_ds = ArXivDataset(titles_test, summaries_test, tags_test, possible_tags, base_tokenizer)\n",458 "\n",459 "base_train_dl = DataLoader(base_train_ds, shuffle=True, \n",460 " batch_size=128, num_workers=0)\n",461 "base_test_dl = DataLoader(base_test_ds, shuffle=True, \n",462 " batch_size=128, num_workers=0)\n",463 "\n",464 "small_ds = ArXivDataset(titles_train[:256], summaries_train[:256], tags_train[:256], possible_tags, base_tokenizer)\n",465 "small_dl = DataLoader(small_ds, shuffle=True, \n",466 " batch_size=128, num_workers=0)"467 ]468 },469 {470 "cell_type": "markdown",471 "metadata": {472 "cellId": "61ne5cjsd7uuxa67duqn1g"473 },474 "source": [475 "Следующий класс в итоге не используется, мы будем использовать встроенную архитектуру головы классификатора."476 ]477 },478 {479 "cell_type": "code",480 "execution_count": 12,481 "metadata": {482 "cellId": "wmqvly8ahucv5xfnudiaj"483 },484 "outputs": [],485 "source": [486 "#!g1.1\n",487 "import torch.nn as nn\n",488 "import torch.nn.functional as F\n",489 "\n",490 "class BaseArXivClassifier(nn.Module):\n",491 " def __init__(self):\n",492 " super(BaseArXivClassifier, self).__init__()\n",493 " self.head = nn.Sequential(\n",494 " nn.Flatten(),\n",495 " nn.Linear(512*768, 155),\n",496 " nn.Softmax(dim=1)\n",497 " )\n",498 " \n",499 " def forward(self, X):\n",500 " with torch.no_grad():\n",501 " X = base_model(X).last_hidden_state\n",502 " X = self.head(X)\n",503 " return X"504 ]505 },506 {507 "cell_type": "markdown",508 "metadata": {509 "cellId": "mavysgsu3gr19liok5m4p3"510 },511 "source": [512 "Определим функции для тренировки и отрисовки графиков"513 ]514 },515 {516 "cell_type": "code",517 "execution_count": 13,518 "metadata": {519 "cellId": "5dbnf9xl7hif6k1uzavgjr"520 },521 "outputs": [],522 "source": [523 "import matplotlib.pyplot as plt\n",524 "\n",525 "def plot_learning_process(train_loss, val_loss): \n",526 " plt.figure()\n",527 " plt.title('loss by epoch')\n",528 " plt.plot(np.arange(0, len(val_loss))+0.5, train_loss, label='train')\n",529 " plt.plot(np.arange(0, len(val_loss))+1, val_loss, label='val')\n",530 " plt.legend()\n",531 " plt.grid()\n",532 " plt.xlabel('epoch')\n",533 " plt.ylabel('loss function')\n",534 " plt.show()\n",535 " "536 ]537 },538 {539 "cell_type": "code",540 "execution_count": 14,541 "metadata": {542 "cellId": "441ify9vq4qumr997qqum"543 },544 "outputs": [],545 "source": [546 "#!g1.1\n",547 "\n",548 "device = 'cuda'\n",549 "\n",550 "import IPython\n",551 "from math import ceil\n",552 "\n",553 "\n",554 "def train_loop(model, dataloader, loss_fn, optimizer, step=0.05, history_loss=None):\n",555 " out = display(IPython.display.Pretty('Learning...'), display_id=True)\n",556 " \n",557 " size = len(dataloader.dataset) \n",558 " len_size = len(str(size))\n",559 " batches = ceil(size / dataloader.batch_size) - 1\n",560 " \n",561 " train_loss = []\n",562 " percentage = 0\n",563 " for batch, (X, y) in enumerate(tqdm(dataloader, leave=False, desc=\"Batch #\")):\n",564 " X, y = X.to(device), y.to(device)\n",565 " # evaluate\n",566 " output = F.softmax(model(X).logits, dim=1)\n",567 " loss = loss_fn(output, y)\n",568 " train_loss.append(loss.item())\n",569 " \n",570 " # backpropagation\n",571 " optimizer.zero_grad()\n",572 " loss.backward()\n",573 " optimizer.step()\n",574 " # print info\n",575 " if batch / batches > percentage or batch == batches: \n",576 " out.update(f'[{int(percentage * size)}/{size}] Loss: {train_loss[-1]:>8f}')\n",577 " percentage += step\n",578 " \n",579 " if history_loss is not None:\n",580 " history_loss.append(np.mean(train_loss))\n",581 "\n",582 " return {'train_loss': np.mean(train_loss)}\n",583 " \n",584 "def test_loop(model, dataloader, loss_fn, history_loss=None, history_acc=None):\n",585 "\n",586 " size = len(dataloader.dataset)\n",587 " test_loss, correct = 0, 0\n",588 " batches = ceil(size / dataloader.batch_size)\n",589 "\n",590 " val_loss = []\n",591 " \n",592 " with torch.no_grad():\n",593 " # evalute and check predictions\n",594 " for batch, (X, y) in enumerate(tqdm(dataloader, leave=False, desc='Batch #')):\n",595 " X, y = X.to(device), y.to(device)\n",596 " output = F.softmax(model(X).logits, dim=1)\n",597 " loss = loss_fn(output, y)\n",598 " test_loss += loss.item()\n",599 " \n",600 " val_loss.append(loss.item())\n",601 " \n",602 " test_loss /= batches\n",603 " correct /= size\n",604 " \n",605 " print(f\"Validation accuracy: {(100*correct):>0.1f}%, Validation loss: {test_loss:>8f} \\n\")\n",606 " \n",607 " if history_loss is not None:\n",608 " history_loss.append(np.mean(val_loss))\n",609 " \n",610 " return {'val_loss': np.mean(val_loss)}"611 ]612 },613 {614 "cell_type": "markdown",615 "metadata": {616 "cellId": "eihlww9xq7sp8o0j4al6ir"617 },618 "source": [619 "И натренируем нашу базовую модель."620 ]621 },622 {623 "cell_type": "code",624 "execution_count": 338,625 "metadata": {626 "cellId": "vspefy8lcknk40w5yxt",627 "collapsed": true,628 "jupyter": {629 "outputs_hidden": true630 }631 },632 "outputs": [633 {634 "data": {635 "application/vnd.jupyter.widget-view+json": {636 "model_id": "ea9879f7935b4807a7d5bd38c183a3ef",637 "version_major": 2,638 "version_minor": 0639 },640 "text/plain": [641 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=483.0), HTML(value='')))"642 ]643 },644 "metadata": {},645 "output_type": "display_data"646 },647 {648 "data": {649 "application/vnd.jupyter.widget-view+json": {650 "model_id": "b7227056dcac4587b0eac589ee7b0f51",651 "version_major": 2,652 "version_minor": 0653 },654 "text/plain": [655 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=267967963.0), HTML(value='')))"656 ]657 },658 "metadata": {},659 "output_type": "display_data"660 },661 {662 "name": "stderr",663 "output_type": "stream",664 "text": [665 "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n",666 "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",667 "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",668 "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'pre_classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",669 "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"670 ]671 },672 {673 "name": "stdout",674 "output_type": "stream",675 "text": [676 "\n",677 "\n",678 "Epoch 1\n",679 "-------------------------------\n"680 ]681 },682 {683 "data": {684 "text/plain": [685 "Learning..."686 ]687 },688 "metadata": {},689 "output_type": "display_data"690 },691 {692 "data": {693 "application/vnd.jupyter.widget-view+json": {694 "model_id": "907aa42b0a7746b0898b9bd46a9e0cf2",695 "version_major": 2,696 "version_minor": 0697 },698 "text/plain": [699 "HBox(children=(HTML(value='Batch #'), FloatProgress(value=0.0, max=289.0), HTML(value='')))"700 ]701 },702 "metadata": {},703 "output_type": "display_data"704 },705 {706 "data": {707 "text/plain": [708 "'[0/36900] Loss: 0.038880'"709 ]710 },711 "metadata": {},712 "output_type": "display_data"713 },714 {715 "data": {716 "text/plain": [717 "'[1845/36900] Loss: 0.034217'"718 ]719 },720 "metadata": {},721 "output_type": "display_data"722 },723 {724 "data": {725 "text/plain": [726 "'[3690/36900] Loss: 0.029051'"727 ]728 },729 "metadata": {},730 "output_type": "display_data"731 },732 {733 "data": {734 "text/plain": [735 "'[5535/36900] Loss: 0.021646'"736 ]737 },738 "metadata": {},739 "output_type": "display_data"740 },741 {742 "data": {743 "text/plain": [744 "'[7380/36900] Loss: 0.022256'"745 ]746 },747 "metadata": {},748 "output_type": "display_data"749 },750 {751 "data": {752 "text/plain": [753 "'[9225/36900] Loss: 0.023418'"754 ]755 },756 "metadata": {},757 "output_type": "display_data"758 },759 {760 "data": {761 "text/plain": [762 "'[11070/36900] Loss: 0.021545'"763 ]764 },765 "metadata": {},766 "output_type": "display_data"767 },768 {769 "data": {770 "text/plain": [771 "'[12915/36900] Loss: 0.021974'"772 ]773 },774 "metadata": {},775 "output_type": "display_data"776 },777 {778 "data": {779 "text/plain": [780 "'[14759/36900] Loss: 0.021328'"781 ]782 },783 "metadata": {},784 "output_type": "display_data"785 },786 {787 "data": {788 "text/plain": [789 "'[16605/36900] Loss: 0.020796'"790 ]791 },792 "metadata": {},793 "output_type": "display_data"794 },795 {796 "ename": "KeyboardInterrupt",797 "evalue": "",798 "output_type": "error",799 "traceback": [800 "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",801 "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",802 "\u001b[0;32m<ipython-input-2-fabbf86ed4e8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Epoch {epoch+1}\\n-------------------------------\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mtrain_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbase_train_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 22\u001b[0m \u001b[0mtest_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbase_test_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",803 "\u001b[0;32m<ipython-input-1-09a3ee852cb4>\u001b[0m in \u001b[0;36mtrain_loop\u001b[0;34m(model, dataloader, loss_fn, optimizer, step, history_loss)\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;31m# backpropagation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",804 "\u001b[0;31mKeyboardInterrupt\u001b[0m: "805 ]806 }807 ],808 "source": [809 "#!g1.1\n",810 "import torch.nn as nn\n",811 "import torch.nn.functional as F\n",812 "\n",813 "from IPython.display import clear_output\n",814 "\n",815 "classifier = DistilBertForSequenceClassification.from_pretrained(base_model_name, num_labels=num_tags).to('cuda')\n",816 "for param in classifier.base_model.parameters():\n",817 " param.requires_grad = False\n",818 "\n",819 "classifier.train()\n",820 "loss_fn = F.binary_cross_entropy\n",821 "optimizer = torch.optim.Adam(classifier.parameters(), lr=1e-4)\n",822 "epochs = 30\n",823 "\n",824 "train_loss = []\n",825 "val_loss = []\n",826 " \n",827 "for epoch in range(epochs):\n",828 " print(f\"Epoch {epoch+1}\\n-------------------------------\")\n",829 " \n",830 " train_loop(classifier, base_train_dl, loss_fn, optimizer, history_loss=train_loss)\n",831 " test_loop(classifier, base_test_dl, loss_fn, history_loss=val_loss)\n",832 " \n",833 " clear_output()\n",834 " plot_learning_process(train_loss, val_loss)"835 ]836 },837 {838 "attachments": {839 "72b0e3ec-070b-4ffe-a261-f72c4a5740fe.png": {840 "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAgAElEQVR4nO3deXxU5b3H8c9kMlkmkz1kIQsJCTthC5uASEAExAVXRL3a2qu2tXVpbbW1rXpdatVeW/W6XJe6gZHiVURBBE1EAdl3EsAgJAGSQCCBkIQkk7l/HGRRIDnZJpn5vl+veQVmzpz8nonmy3POs1gAFyIiIk3k4+4CRESkc1FwiIiIKQoOERExRcEhIiKmKDhERMQUBYeIiJii4BAREVMUHCIiYoqCQ0RETFFwiIiIKQoOERExRcEhIiKmKDhERMQUBYeIiJii4BAREVMUHCIiYoqCQ0RETFFwiIiIKQoOERExRcEhIiKmKDhERMQUX3cX0B4iIyNJTk5u9LijR48SFBTUDhW1PU9qC6g9HZkntQU8qz0tbcuaNWvO+prL0x8ZGRmupsjOzm7ScZ2BJ7XF5VJ7OjJPaovL5VntaWlbzvY7VZeqRETEFAWHiIiYouAQERFTvOLmuIiIWXV1dRQVFVFTU+PuUpotNDSU3NzcRo8LCAggISEBm83WpPMqOEREzqCoqIjg4GCSk5OxWCzuLqdZjhw5QnBw8DmPcblclJWVUVRUREpKSpPOa8G4S+7RYmJiyMrKavS4yspKHA5HO1TU9jypLaD2dGSe1BY42Z7Q0FBSU1M7bWgAOJ1OrFZro8e5XC7y8/OpqKg47fnMzMyzv8fTHxqO2/mpPR2XJ7XF5TrZnq1bt7q5kpY7fPhwk489U3vP9jtVN8fP4a3lu/how153lyEi0qEoOM7hvVWFfLC2yN1liIgXKi8v54UXXjD9vosvvpjy8vI2qOgkBcc5JIQHUnSo2t1liIgXOltw1NfXn/N98+fPJywsrK3KAjSq6pwSwu0s2X4Al8vVqW+QiUjnc//995Ofn8+gQYOw2WwEBAQQHh5OXl4e27dvZ9q0aRQWFlJTU8Ndd93FbbfdBkBycjKrV6+msrKSSZMmMXbsWJYtW0Z8fDxz584lMDCwxbUpOM4hITyQ6jonB4/WEunwd3c5IuImD8/bwta9h1v1nH27hvDgpf3O+voTTzzB5s2bWb9+PTk5OUydOpXNmzefGDL7+uuvExERQXV1NcOGDeOqq64iMjLytHPk5+fz3nvv8corr3Dttdfy/vvvc+ONN7a4dl2qOoeEcDuALleJiNsNHz78tHkWzz77LAMHDmTkyJEUFhayY8eOH72nW7duDBo0CICMjAx27drVKrWox3EOCeFGl67oUDUDE9v2mqGIdFzn6hm0l1OXR8/JyWHx4sUsX74cu93OuHHjzjjD3d//5JUSq9VKdXXr/CNYPY5ziD8RHFVurkREvE1wcDBHjhw542sVFRWEh4djt9vJy8vjm2++adfa1OM4h5AAG6GBNl2qEpF2FxkZyejRo+nfvz+BgYHExMSceG3y5Mm89NJL9OnTh169ejFy5Mh2rU3B0QhjSK56HCLS/mbNmnXG5/39/VmwYMEZX/v+PkZUVBQrVqw48fy9997banXpUlUjNJdDROR0Co5GJITbKTpUjbFsi4iIKDgacepcDhERUXA0SnM5REROp+BoxKlzOURERMHRKM3lEBE5nYKjEZrLISKdQXvuwqjgaALN5RAROUkTAJsgITyQnfuPursMEfEi999/P4mJidxxxx0APPTQQ/j6+pKdnc2hQ4eoq6vj0Ucf5fLLL2/32hQcTaB9OUS83IL7oXhT654zNh2mPHHWl6dPn87dd999Ijhmz57NwoULufPOOwkJCeHAgQOMHDmSyy67rN1/Lyk4mkD7cohIexs8eDClpaXs3buX/fv3Ex4eTmxsLPfccw9LlizBx8eHPXv2UFJSQmxsbLvWpuBogsRT5nIoOES80Dl6Bm3pmmuuYc6cORQXFzN9+nRmzpzJ/v37WbNmDTabjeTk5DMup97WdHO8CRIijCG5hbpBLiLtaPr06WRlZTFnzhyuueYaKioqiI6OxmazkZ2dze7du91Sl3ocTRAfpkmAItL++vXrx5EjR4iPjycuLo4bbriBSy+9lPT0dIYOHUrv3r3dUpeCowmCA2yE2W0akisi7W7TppM35aOioli+fPkZj6usrGyvknSpqqm0vLqIiEHB0UQJYXYFh4gICo4m+372uPblEPEe3vL/u9l2WgCP/2RiYmLIyspq9LjKysqzrveyaHcdM3NreTbTToh/x58EeK62dEZqT8flSW2Bk+1xOBzExMQQGhraaSf+Op1OrFbrOY9xuVxUVFRQUlLyo/skmZmZZ3yPV9wcT0hIYNy4cY0el5OTc9bj6reWMDN3NUl9BzMoMay1S2x152pLZ6T2dFye1BY42Z66ujqKiorYs2ePu0tqtpqaGgICAho9LiAggIEDB2Kz2Zp0Xq8Ijtbw/VyOokNVnSI4RKRlbDYbKSkp7i6jRXJychg8eHCrn1f3OJpIczlERAwKjibSXA4REYOCwwTN5RARUXCYorkcIiIKDlM0l0NERMFhSkJ4IDV1DZQdrXV3KSIibqPgMCHhlH05RES8lYLDhFPncoiIeCsFhwmayyEiouAwRXM5REQUHKYlhmtIroh4NwWHSZoEKCLeTsFhkuZyiIi3U3CYlBBup6augQOVmsshIt5JwWFSQriG5IqId1NwmKRJgCLi7RQcJsWHay6HiHg3BYdJDn9fwjWXQ0S8mIKjGRI0l0NEvJiCoxm+H5IrIuKNFBzN8P0kQM3lEBFvpOBohoRwO8fqNZdDRLyTgqMZNJdDRLyZgqMZNJdDRLyZgqMZNJdDRLyZgqMZNJdDRLyZgqOZNJdDRLyVBfD4MaUxMTFkZWU1elxlZSUOh6NJ53x+XQ17Khv46/n2lpbXJsy0pTNQezouT2oLeFZ7WtqWzMzMs77m8vRHRkaGqymys7ObdJzL5XI9+vEWV88H5rsaGhqa/J72ZKYtnYHa03F5UltcLs9qT0vbcrbfqbpU1UyayyEi3krB0UyayyEi3krB0UyJEZrLISLeScHRTPFhmsshIt5JwdFMQf6+RAT56VKViHgdBUcLfL9KroiIN1FwtEBCeCCF6nGIiJdRcJjlrIfyQsAYkrtH+3KIiJdRcJiVNQPenQEYPY5j9Q3srzzm5qJERNqPgsOs5DFQsgkqik6Zy6H7HCLiPRQcZvWcbHzd/qn25RARr6TgMCuqJ4SnwLZPT5nLoRvkIuI9FBxmWSzQawp8t4Qgy7HjcznU4xAR76HgaI6ek8B5DHbmaC6HiHgdBUdzJI0C/xDYtuB4cOhSlYh4DwVHc/j6QdoE2PEZCWEBmsshIl5FwdFcPSdDZQkDrd9pLoeIeBUFR3P1uAgsPvQ9sgzQkFwR8R4KjuayR0DiCOJKvgQUHCLiPRQcLdFzEgEHNhNLmW6Qi4jXUHC0RM8pAFwauFE9DhHxGgqOlujSC8KTuch3vYJDRLyGgqMlLBboOZmB9RvYf/CQu6sREWkXCo6W6jkZP1ct3cpXUlPndHc1IiJtTsHRUt1GU+/rYCxr+CKv1N3ViIi0OQVHS/n6Ye0xgYm+65m7ttDd1YiItDkFRyuw9JpCFw6xf8dKKqrq3F2OiEibUnC0hh4X4cLCWNawYPM+d1cjItKmLIDHr84XExNDVlZWo8dVVlbicDia9T0Gr72PfYePcY/9r9w3PLBZ52hNLWlLR6T2dFye1BbwrPa0tC2ZmZlnfc3l6Y+MjAxXU2RnZzfpuDNa8neX68EQ14j733LtK69u/nlaSYva0gGpPR2XJ7XF5fKs9rS0LWf7napLVa2llzGLPNNnHfM27HVzMSIibUfB0Vq69IawJK6wb+LD9XvcXY2ISJtRcLQWiwV6TmGIcwP5e/fzbekRd1ckItImFBytqddkfBuOMca6hbnrdblKRDyTgqM1dRsNfg6uD9vK3PV7tZ2siHgkBUdr8vWH1PGcV7+agoNHWVdY7u6KRERanYKjtfWaQuCxUgb5FvCRLleJiAdScLS2tImAhf+MzuPjjXupdza4uyIRkVal4Ghtji6QMIzzXas5UFnL0vwyd1ckItKqFBxtoddkQg9tITXgMHPXaU6HiHgWBUdb6DkZgF/G57NwSzHVtdrgSUQ8h4KjLUT3hdAkMi1rOVrrZHFuibsrEhFpNQqOtmCxQK/JhBcvo1uwRZMBRcSjKDjaSt9pWOqrua/rOr7cXkp5Va27KxIRaRUKjrbSbRTEZ3DhwSwanPXM31Ts7opERFqFgqOtWCww5h78jhTwk7ANWjFXRDyGgqMt9ZoKkT34ufUjVn5Xxt7yandXJCLSYgqOtuTjA2PupsvR7Yz12chH2uBJRDyAgqOtpV8LwV35XdACja4SEY+g4Ghrvn4w6lek120koHgN20u0wZOIdG4KjvYw5GYaAsL4pW0ec3WTXEQ6OQVHe/B34DPidib6rGbD2hXa4ElEOjUFR3sZfjv11gCmVc1huVbMFZFOTMHRXoIiYfBNTLMu5eV5X2qfDhHptBQc7ch3zK/xsVi4oGw2M1cUuLscEZFmUXC0p7AkLAOu5npbNq9+tooDlcfcXZGIiGkKjnZmGX0PAa5jXONcwBML8txdjoiIaQqO9hbdG3pN5Vb/Rcxf8y1rdh9yd0UiIqZYAI8fGxoTE0NWVlajx1VWVuJwONq8npCKbQxZ93ue5j/4yDaVh0YF4GOxtOr3aK+2tBe1p+PypLaAZ7WnpW3JzMw862suT39kZGS4miI7O7tJx7WK1y92VT3R05V234euN5d91+qnb9e2tAO1p+PypLa4XJ7Vnpa25Wy/U3Wpyl3G3ENgdTH3xm3g6YXbdKNcRDoNBYe7pE2AmHR+6ppLTV0df9ONchHpJBQc7mKxwJi78Sv/lsf7FvHvNUW6US4inYKCw536ToPwZK6ofI/YYH/+MnczzgaPH6sgIp2cgsOdrL4w6k6s+9by4oDtbNlbwawVu91dlYjIOSk43G3QDRCbzuA1f2Be6NN8uHARZbpRLiIdmILD3WwBcGs2THmSvuxktut3bH/9Vjh6wN2ViYickYKjI7DaYMTtWO9az9rYaxhWNo/6fw6CZc9Dfa27qxMROY2CoyOxR9Dnlhe50e8frHf1hM8egBdGQN580OZPItJBKDg6GIe/LzdcMpGrj/yWTwc+Bz6+kDUD3p4GJVvcXZ6IiIKjI7pkQBwX9onhFysj+fC82TDlSdi7Hl4aA0uecnd5IuLlFBwdkMVi4fnrBzMyJZLfzNnKJ4GXwZ3roPclkP047N/m7hJFxIspODqoAJuV134ylIxu4dyVtY7PvquFS/4Bfg5Y/LC7yxMRL6bg6MDsfr68/pNh9I8P5Y5Za8kucsLou2DbJ1DwjbvLExEvpeDo4IIDbLx5y3B6xQbz87fXsDx6OjhiYdFfNNJKRNxCwdEJhAbaePuWEaREBXHLzC18l34nFK6AvE/cXZqIeCEFRycRHuTH2z8bQdewAC5fmkxNaCp8/jA4691dmoh4GQVHJ9Il2J9Zt44kItjO7yuuhAPbYd3b7i5LRLyMgqOTiQkJYNatI1kbcB7r6UXdF49D7VF3lyUiXkTB0Ql1DQvk3dvO4wXbzdiqSilb/A93lyQiXkTB0UklRti5/7abybYMx3/lc3y1PtfdJYmIl1BwdGLduzhIu+5JAjnGjjkP8eDczdTUOd1dloh4OAVHJ5fYazCuQTdys20xX3yzksufX0pe8WF3lyUiHkzB4QF8x/8Rq9XGnJ6fU3a0lsueX8qiXXW4NEFQRNqAgsMThMTBeb8kZvfHLJoRypi0KGbm1fLTN1ax/4i2oRWR1qXg8BSj74LACMKXPsprNw/lxj5+LMsvY8o/l5C9rdTd1YmIB1FweIqAUBj7O9iZgyX/Cy7sZmPer8YQGeTPT/+1ioc+2qIb5yLSKhQcnmTYzyAsCRY/CK4GesUGM/dXo/nJqGTeWLaLaf+zlGX5B9xdpYh0cgoOT+LrD+P/AsWbiC5dAhj7ejx0WT/+9ZNhlFfVcf0rK5jxv9+wetdBNxcrIp2VBfD4oTcxMTFkZWU1elxlZSUOh6MdKmpDrgYy1vwW32OH2JN0JeVh/ah0JIPFSq3TRU5hPR/vrONwrYv0KCtX9rCREmp1d9WN8oifzSk8qT2e1BbwrPa0tC2ZmZlnfN4rgiMjI4PVq1c3elxOTg7jxo1rh4raWNFqqt+5gcCaYuPv/iGQOAKSR0O30VRF9eetlft46ct8yqvqmNg3ht9M7EmfuBD31n0OHvOzOc6T2uNJbQHPak9L22KxWM74vG+zzygdV8JQVox8mXGDe0DBctj1NexeBosXAWD3DeTnicP46aiRfFzRnYc3+jBlawlTB8Rxz4U9SIsOdnMDRKQjU3B4stB4SL/aeABU7jeCZPcy2L0U/6+f4ipcXJY6kRei/sjL35SyYNM+pg2K564Le9AtMsi99YtIh6Tg8CaOLtD3MuMBUF0OG97FtvAB7jq6j5t+/hYvrDvGW8t389GGvcwYnsSvJ6QRHRzg3rpFpEPRqCpvFhgGI38BN86B8kLCZ07mgYHVLPl9JtOHJTJrZQEXPJnD0wu3cbimzt3VikgHoeAQSB0PP/sMbIHwxsXEFH7KY1eks/g3F3Bh3xiez/6WsU9m88qSnZpEKCIKDjkuujfc+gXEDYJ/3wxLniYl0s5zMwbz8a/HMCAhjMfm55L5dA7vrSqg3tng7opFxE0UHHJSUBTcNBfSr4UvHoEPfwn1x+gfH8pbtwxn1q0jiAkJ4L73NzHpH0v4dPM+rcAr4oV0c1xOZwuAK/8XItMg53E4tAumvwNBkYxKjeKDX0by2dYSnlq4jZ+/s5bescFcPiieqelxJEXa3V29iLQD9TjkxywWGHcfXPUa7FkDr06AAzuOv2RhUr9YFt49lqeuHkCAzcrfPs1j7FPZXPrc17z0ZT6FB6vc3AARaUvqccjZpV9tLJr47gwjPK58FXpeBIDVx8I1QxO5ZmgihQerWLB5H59s3McTC/J4YkEeAxNCmTogjovT40gIV09ExJOoxyHnljjcuGke3BVmXQMzr4X9204/JMLObWNTmfurMXz1+0z+MKU3LuDx+XmM+Vs2l//PUl5ZslM9EREPoR6HNC68G9yWAytfhiVPwwvnwdCfwrg/GDfUT5EYYef2C1K5/YJUCsqq+GTTPj7ZtJfH5ufy2Pxc+saFMLl/LJP7x9Ij2nHWtXBEpONScEjT2AKMXQYH3QA5T8Dq12HjbDj/tzDi58brP5AUaecX41L5xTgjRBZuKebTLcX896Lt/Pei7aREBTGpnxEiA+JD8fFRiIh0BgoOMScoCqY+DcNvhUV/MTaNWvUaTHwI+l1p3Fg/g6RIO7eO7c6tY7tTeriGz7aWsHBLMa9+tZOXvswnNiSASf1imNQ/luHJEfhadRVVpKNScEjzdOkF178HO3Ng4Z9gzi3wzYsw6XHjvsg5RIcEcOPIbtw4shsVVXV8nlfCp5uLyVpVyJvLdxMZ5MdF/WK5ZEAcI1IUIiIdjYJDWqb7OLj9S9jwLnz+CLw2EfpcZly+6jbqrD2Q74XabVw5JIErhyRQVVvPl9v288mmfcxdv4d3VxYQEeTHpH6xJDQ4GeNsUIiIdAAKDmk5HysMvhH6ToNlz8E3L0DuRxDVEzJ+AgNngD2i0dPY/XyZkh7HlPQ4qmudfLm9lPkb97Bj/Vfk1Dp4LfdzJvWLYWp6V0Z2V09ExF0UHNJ6/B2Q+QcYfSds+QDWvAEL/wiLH4a+lxsh0lgvxOWCgzsJ3JnN5J05TC5YAj4VOAOsfOOYwoPrJ/HuykIigvwY3zua0WmRnNc9ithQLf0u0l4UHNL6/IKMHsjgG6F4M6x9Eza8B5tmn7kXcrQMvvsSdmZDfg5UFBjPhyYal71SxrJvxYeMLv6MRb6fUpR6Ja9aruDDrSXMWVMEQPeoIM5LNZZFGdk9gkiHv5saL+L5FBzStmL7w8VPwYUPH++F/OtkL6THRKgohH0bARf4h0LK+UaPJXU8RHQ/0Tv59mA0CdOfxvL1MySueYOHXf/Hg4NvYEfP21hSamf5zjI+XLeHmSuM0OkdG3wiSIanRBAaaHPjhyDiWRQc0j787DD4BuPxfS8kdx5EpELmA5CaaSzpbj3Hf5IhXY0QGnMPfP0MPmveoNf6mfQadAO3Xv5b6kIy2LSnguX5ZSzPL2PWigL+tXQXPhYY2i2C8X2imdA7mjRNPBRpEQWHtL/veyEXP9W89/8gQFjzBqyfiW3QDQwZcw9DMtO4IzONY/VO1heU8/W3B/g8t/TEOlqJEYFM6B3D+N7RjOgegb+vtXXbJ+LhFBzSeZ0pQNa+CTH9Ie1C/HtMZES3EYzoHslvL+rFvopqsvP280VeCVmrCnhj2S7sflbGpEUxoU80mb2iiQ455SZ77VEo2QKB4cYy8+qliAAKDvEEpwbIxtnw7WJY/jws/Qf4h0D3CyBtInE9JnL9iCSuH5FETZ2T5fllfJ5Xwhe5pXy5tZA+lgLGhxRxXmAhPep3EHp0JxbX8Z0OQ5MgbTykTjDOFxDa9u0q2QqVJWCPNGbs2yPBVzf9xf0UHOI5QrrCmLuNR81hY6TWjkVGkOTOM46J7gc9LiSg2xgyK4vI9FmHK3wd1OViaaiHY1B2LJQ1zu5sck0j35rGkIgazrdsJHnDv/Fd8wZYrMbs+NQJRpjEDQafVpxT0tAAX/8dvngM+MEOi37BEBQJ9qhTAiUCkkZB74tbrwaRc1BwiGcKCIE+lxoPlwtKc+HbRUaQLH8Blv7TOC4wAkvXwTB6EnQdDF0HExEcR0pZFeUF5ewvOMTsgnIe2TcSH9ctDLZ8y7TgrYwt3URiwaOQ/SiuwEgsqZnHv99lLQuR6kPwf7fDjoWQfo0xdLnqIFQdgKoyY+jy938+ss+4lFZ1wJh4efXr0P+q1vn8RM5BwSGez2KBmL7GY/RdcOwI7F0HYd2Mjap+cO/CAnTv4qB7FwdXZSQAUHmsno2F5awt6MvnBWN5pqgcV81+xvhs4gLnRsZtWUzE5jkUhwxg3+hHSO4/ivAgP3N17l0Ps2+Cw3vh4qdh2H827b5K/TF463Jjj/iwbpAw1Nz3FTFJwSHexz8YUsaaeovD35dRaVGMSjP2H3G5XJQeOcaWvRPYvOcwfyg6RFLhR9xW8SYD508ja14m7wTdTHx8An3iQugbF0zv2BCSIuxnXj5+7Vvwyb3GpaefLoDEYU0vztcfps+EV8fDu9cZG2+FJZlq34/s2wDhKUbPTeQHLPzoIqrniYmJISsrq9HjKisrcTgc7VBR2/OktkDnaU9NVSWx+Vmkl31CtSWQF7mWl6vHU48x5NffCgkOH2IDnaRG+JMcVMfE0tdILFnMwfCB5Pb5LXV+zbvxbj9ayJC191ETEMW6wU/g9G3Glr0uF4mFH5C6800OhQ1gw8D/arTX01l+Nk3lSe1paVsyMzPP+LxXBEdGRgarV69u9LicnBzGjRvXDhW1PU9qC3TC9pTmwYLfw3df0hDdj53DHmK1qzd5xUfYuu8wmwsPEuks4UXbP+nvs4s3fK/hi9hbiI8IJiE88JSHnS4O/6ZvcpWfDe9cBWkTYEaWsQBlUzU4jZpXvQpd+sD+XLjiZRh43Tnf1ul+No3wpPa0tC1nmyirS1UibSG6N9w0F3I/wmfhA6R9cg1p/a+Gix6BkH5snPMk/Xc8j7PBxbye/2BV3UAqDlaxZUsxZUdrTzuVn9WHrmEBJITbSQgPpEdMMMOSw+kbF/LjFYJTM42hyZ/8Bj77E0z+a9Pqra2C938G2+bDqDthwoPwrynw6R8gbaIxkkvkOAWHSFuxWIxVgdMmGnNKvv4HbFsAPSYyYOuHEDsAn+lvc2l4Mpee8raq2nr2HKqmqLyaokPVFB2qouhQNXsOVbM4t4SsVYUABPlZGdItnGHJEQxLjmBwUhgBNisM+xmUfWssbx+ZZvz9XCr3w7vTjQEDFx/f3RHg0n/Cy+cbAXTFi23zGUmnpOAQaWt+dsj8Iwy6HhY+AFs/ZG/cRLre8s4Z92q3+/nSIyaYHjHBZzxdcUUNK3cdZNV3B1m16yDPLN6OywU2q4X0+FCGpUQwPOnXjCndgd/832GJSDEWjTyTA9/CzKvgSAlMfwd6Tz352vej0L76u3G5qvsFrfFpiAdQcIi0l/BkuG4mVJayffVWup4hNJoiNjSAywZ25bKBXQGoqKpj9e6DJ8Lk9a+/42WniyCuY45fHvFv3cDNPo9R6p+Mw9+XIH8rQf6+pDfkcUfxn8Fi4ethr5IYcj49f7jL4tjfGasaf3w3/GIZ2AJb45M4s9qjUFkK1QeNZWM0S77DUnCItDdHNLC11U4XarcxoU8ME/rEAFBd62R9YTl5xYdZWfECV627mdcsT/FM4ouUOO1U1TrpX5HDXYefpsQSyW3O+8nLtkD2VwTarKTHhzIwMZRBieEMTAwlfuozWN6+HJY8DRP+3LwinfXGTP7De4xlVCpLf/y1tvLk8bEDYMa7EJrQCp+QtDYFh4iHCfSzcl5qJOelRgIpMODf8MZUHjn2hHHDftVrsPCvkDCMpBlZLLBHsLusig1F5awrKGdDUTlvLtvNK87vAIhy+POsYyIjvn6GNY5MuvYcQnxYYNOXpq+tgjk/he2fnnwuIAwcMUaIdh1ifHVEG885a+GzP8P/ZsJ1s8zNaZF2oeAQ8XQJQ2Hai8Yv75fGwIHt0PsSuOpVsAViAZKjgkiOCuLyQfEA1NY3kLvvMBuKyllfUM6TBTfxesNyrJ/czfkfPIjdz0ZaTDA9ox34VNbhiiulZ0wwXUMDTg+UqoMwazoUrYJJfzWWZQnqcsZ7O6dJHGlMZnxjKlz2bKNDgqV9KThEvEH/K42RVtmPwYhfwKTHzjnHw8/Xh4GJYQxMDOOm8wAGUbXyMBnz7yBrSC7z/S9mR2kl2dv2c6Cylve2rQKMGfZp0Q7Soh30DzrMVVvvJKh6D9XT/oZ81t8AAAfxSURBVEXQoCuaXm90b2MG/Oyb4IPbjbXGJvzF3LwUaTMKDhFvMfZ3xl7vYYnNert92A2QO5sR+c8x4o7/gJD+AHz8WTbRPQayveQI35ZWsr3kCHu2rebeukdwcYzran/Pyiw/Qud+RnKknaTIILpF2EmKtJMUYSc00Ibdz0qgnxW7ny92m9WY8GiPgP/4ABbcZwxn3r8NrnrFWDJG3ErBIeItLJZmh8aJ91/yDLw4yphhPv1tABx+FoanRDA8JcI4btdSePdhXAF2dk2ZzX+SxMSyKnYfPGrcSyksZ/6mfTgbzr5oRYDNB7ufL4E2K0H+l3Olw59bt79E2TPnkz3kWUK79iAh3E5iuJ2QQF9tBdzOFBwi0nSRqXDB7+Hz/4K8+T/eAyR3Hsz5GYR3w3Lj+6SEJZFyhtPUORvYW15NwcEqKmvqqap1UlXnpLq2nqPHnFTXOamqrafqmJOqWidLa6exy9WVP1T+lYlLZ/CL2ntY4eoDQLC/L/HHl2dJCA8kMcJOcqSdFHsNCbX5+O3fCjUVMPw2cHRp+WdQV22MMNu7zpijYwsyhin7BYHNfvw5+8k/B4RBl97GfjEtDbjKUmMpfXskxKa7bVdKBYeImDPqTtg0B+bfCynnn3x+1WvGc/EZcP1s41LTWdisPnSLDKJbZJCJbzwC14EpNMyaTtahv7Jl8J/5Jvwy9h48Qn3pduwluXTZuYPUht308dlNjKX8xDsbsFC99GVW9PoddX2vpltUEInhdmzH56y4ji/Z53JBrdNFTZ3ztO/s7+tj9GoKvoG5dxj3i2LT4XA91B01Ro7VVRt/PpuAUGMjsZi+EN0XYvpBdJ8z7yZZVwMHthkhUbIFSjYbX4/uP3lMdF/j0uOAayE41sTn2HIKDhExx2ozliN57SJjl8KASZD9OHz5N+g5Ga7+l/Ev7TZgiUrDeuvnMOcW+q99kP5RM+HQbnAeA8Dl64czsicVIRPY5N+dbSSz7lg8FWXF3Hbo74zf+ic+3/Q+P6m7hWLOsf7Wok9P+2tMQAN/DnqfqUc/pCowjpIpM+k6ZIqxxMupXK7jAVJlPGqrjE23Srcaj5ItxvbGxw6ffE9oohECUT1Obs51YAe4joeXb4ARMD0nGRMjo/sawbXhXVj0Z1j8oLEywMAZxsz/tpykeZyCQ0TMSxxurIG18mX6R6yFshUw+Ea45J9gbeNfK4FhRo9myZOwZ40RVrHpENMPS1RPfK02IoFIIB24+vv3NVxL9dcvkLnkMb7yv59l3e9ic+wVYDl9ocjvvttJ9+7dAbBgIaZ8LWO3PkTU0SLedU3k0UPXcfQDCz4ffkpKVBB94kKOP4w9V+JCA7H8MDiTR5/8s8sFFYXGnvKlW45/3Qr5X0BInBEOfS41eiQx/SGi+49Hk3W/wPj8D+wwAmTDe8Yilf4h0G8aDLwekka24od+OgWHiDTPhL9A3idEla2A8++F8X9qv2vuVl9j/S8zfKwEjv019L8En3l3csH2x7mg9iuj9xSZeuKwHEsR48alGb2FLx6B9S8agwqu+4jpyWM572AVufsOk1t85MRcl4837jvx/iA/K6nRDtK6OEiNdtDj+PDkpAi7sZyLxWJstBWWBL0mn6zP5TL/+UX1MH4OmX+CXV8ZIbLpfWNjsPAUuoWOhJrBZ74c1gIKDhFpnoBQuH42m7+eT/8J97u7mqaLSIGbPjJ+uX72J3hxNIx/AEb+8uS/7Hcvh7m/hIM7jS18L3wY/B34cHKy5JT0uBOnPFxTx/biI+QVG0OS8/dXsiy/jP9bt+fEMX5WH5Kj7PSIDiYp0o6zwWUMAKh1Ul3rPPm17uRz1XVO4kIDGRAfSnpCKAMSQukdG4Kf7w+W0/fxMXoh3S8wVjjOnQcbZpFY8BH4/Herf4QKDhFpvrgBHOhy0N1VmGexQMbN0GMifHx875ItH8CUp0jb8SrkfGz0CG6e16RthkMCbAxNjmBo8ukDAo7U1JG//yjfllayo/QI+aWVbNlbwadbirFZLSeGHNv9rCfmsnRx+BvP+1kJsPlQcLCaz7YW895qYzl9P6sPveOCSY83giQ9PoyeMY6Ti1P6O2DQDBg0g+WLP+H8NrjfpOAQEe8V0tVYTHHz+8bclFfHkwDG0N0JDxq/hFsgOMDGoMQwBiWGnfa8y+UyNffE5XJRdKiajUUVbNxTzqaiCj5av5eZKwoAY9TX0ORw3vnZiNPO6/Q1M2qt6RQcIuLdLBZIvxq6j4Nlz7G+MopBF/+6jb+luXsZFouFxAg7iRF2pg4wLpE1NLjYfbCKjUXlbCyqoLa+od0mQio4REQAgqJg4sOU5+S4u5Im8fGxkBIVRMopi1O22/du1+8mIiKdnoJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUyzA2fdvFBER+QH1OERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERMUXCIiIgpCg4RETFFwSEiIqYoOERExBQFh4iImKLgEBERUxQcIiJiioJDRERM+X/cJo73+3EcRgAAAABJRU5ErkJggg=="841 }842 },843 "cell_type": "markdown",844 "metadata": {845 "cellId": "a3sc3gned3uygxq5ywd529"846 },847 "source": [848 "Прошу прощения, исходный output ячейки потерялся, потому что я случайно перезапустил эту ячейку когда тренировал следующую версию модели. Тем не менее, у меня сохранился график обучения, который я прилагаю здесь.\n",849 ""850 ]851 },852 {853 "cell_type": "markdown",854 "metadata": {855 "cellId": "gkqtjsnon6biela1hxs3z"856 },857 "source": [858 "Сохраним ее чтобы иметь возможность использовать ее в интерфейсе."859 ]860 },861 {862 "cell_type": "code",863 "execution_count": 66,864 "metadata": {865 "cellId": "pweno0g9r3w2a22ziovd"866 },867 "outputs": [],868 "source": [869 "#!g1.1\n",870 "classifier.save_pretrained('./models')"871 ]872 },873 {874 "cell_type": "markdown",875 "metadata": {876 "cellId": "tiegy2nj0ccm3c8ec8a2jj"877 },878 "source": [879 "Теперь попробуем повторить историю со специализированной моделью той же архитктуры, но предобученной на научных статьях."880 ]881 },882 {883 "cell_type": "code",884 "execution_count": 312,885 "metadata": {886 "cellId": "my2w2e2vxpnb396jrppeoa"887 },888 "outputs": [889 {890 "data": {891 "application/vnd.jupyter.widget-view+json": {892 "model_id": "516dd347c0544191b3e448f1467ccb70",893 "version_major": 2,894 "version_minor": 0895 },896 "text/plain": [897 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=385.0), HTML(value='')))"898 ]899 },900 "metadata": {},901 "output_type": "display_data"902 },903 {904 "data": {905 "application/vnd.jupyter.widget-view+json": {906 "model_id": "70064bb0aef5416db5980f8cca51fdae",907 "version_major": 2,908 "version_minor": 0909 },910 "text/plain": [911 "HBox(children=(HTML(value='Downloading'), FloatProgress(value=0.0, max=227845.0), HTML(value='')))"912 ]913 },914 "metadata": {},915 "output_type": "display_data"916 },917 {918 "name": "stdout",919 "output_type": "stream",920 "text": [921 "\n",922 "\n"923 ]924 }925 ],926 "source": [927 "#!g1.1\n",928 "sci_model_name = 'allenai/scibert_scivocab_uncased'\n",929 "sci_tokenizer = AutoTokenizer.from_pretrained(sci_model_name)"930 ]931 },932 {933 "cell_type": "code",934 "execution_count": 317,935 "metadata": {936 "cellId": "sclvuck179m0oe02zkimp5"937 },938 "outputs": [],939 "source": [940 "#!g1.1\n",941 "sci_train_ds = ArXivDataset(titles_train, summaries_train, tags_train, possible_tags, sci_tokenizer)\n",942 "sci_test_ds = ArXivDataset(titles_test, summaries_test, tags_test, possible_tags, sci_tokenizer)\n",943 "\n",944 "sci_train_dl = DataLoader(sci_train_ds, shuffle=True, \n",945 " batch_size=256, num_workers=0)\n",946 "sci_test_dl = DataLoader(sci_test_ds, shuffle=True, \n",947 " batch_size=256, num_workers=0)"948 ]949 },950 {951 "cell_type": "code",952 "execution_count": 318,953 "metadata": {954 "cellId": "44aiscms3qxnuo03p2puv9"955 },956 "outputs": [957 {958 "data": {959 "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1DklEQVR4nO3deXxV9Z3/8dcnuTf7RtYLBAFBBRIUZHHBKlWRpG6dUavWWqd1ht9SZ3S0M8VOx7YO/Y12puOMv9r6s9VWrXWp1pG6gBuxLqgsohBACQgStkAgkIXsn98f5yTchCw35N57kpvP8/E4j3vvueec+zkuvPl+v+d8j6gqxhhjTKjivC7AGGPM8GLBYYwxZkAsOIwxxgyIBYcxxpgBseAwxhgzIBYcxhhjBsSCw8Q0EdkuIhdH6bd+KyJLovFb4SYi80Wk0us6zPBgwWGMMWZALDiMMcYMiAWHGTFEJFFE/lNEdrvLf4pIovtdroi8KCI1InJQRN4WkTj3u++JyC4RqRWRT0Xkoj5+JldEXnO3fUtExrvHeEBEftatnqUi8ve91DrFPc5B9ze/FvTdb0XkwZ5+x/3+XBFZJSKH3ddzg77LFpHfuOd/SET+u9vv3iEiVSKyR0S+Ffo/XTOSWHCYkeSfgLOBGcAZwFzgB+53dwCVQB5QAHwfUBE5DbgFmKOq6cBCYHsfv3ED8C9ALrAOeMJd/yhwfVAY5QIXA7/vfgARSQVec7/LB64DfiEi0/r7HRHJBl4C7gdygP8AXhKRHHe/x4EUoMg99n1BxwwAmcBY4GbgAREZ1ce5mhHKgsOMJDcAd6tqlaruB34M3Oh+1wKMBsaraouqvq3ORG5tQCIwTUT8qrpdVbf28RsvqeqfVbUJJ6jOEZFxqvohcBjoaK1cB5Sp6r4ejnEZsF1Vf6Oqrar6EfAccE1/vwNcCmxR1cfdfZ8ENgOXi8hooBT4n6p6yD3Pt4KO2eL+82lR1ZeBOuC0fv+pmhHHgsOMJGOAHUGfd7jrAP4NqABeFZFtIrIYQFUrgNuAHwFVIvKUiIyhdzs73qhqHXAw6DceBb7hvv8Gzt/+ezIeOMvtNqsRkRqc0AuE8Dvdz7HjPMcC44CDqnqol9+tVtXWoM8NQFov25oRzILDjCS7cf5Q7nCSuw5VrVXVO1T1ZOAK4PaOsQxV/b2qnufuq8C9ffzGuI43IpIGZHf8BvA74EoROQOYCvx3L8fYCbylqllBS5qq/q8Qfqf7OXac5y73uNkiktVH/cb0y4LDjCRPAj8QkTx3jOEunD/MEZHLRGSyiAhOl1Ib0C4ip4nIhe4geiNwFGjv4ze+IiLniUgCzhjE+6q6E0BVK4FVOC2N51T1aC/HeBE4VURuFBG/u8wRkakh/M7L7r5fFxGfiFwLTANeVNU9wCs44yWj3OOeP9B/iMZYcJiRZAmwGvgEWA+sddcBnAK8jtOvvxL4haquwBnfuAc4AOzFGVC+s4/f+D3wQ5yuo1kc65rq8Cgwnd67qVDVWuASnHGQ3e7v3uvW0ufvqGo1zhjJHUA18I/AZap6wN3vRpyxjM1AFU43nDEDIvYgJ2Oix/0b/u9wBuFP6H8+EfktUKmqP+hvW2MiwVocxkSJiPiBW4Ffn2hoGDMUWHAYEwXu+EQNziW//+lpMcYMknVVGWOMGRBrcRhjjBkQn9cFRENubq5OmDDB6zJCUl9fT2pqqtdlRISd2/AVy+dn59a7NWvWHFDVvO7rR0RwTJgwgdWrV3tdRkjKysqYP3++12VEhJ3b8BXL52fn1jsR6T4LAWBdVcYYYwbIgsMYY8yARDQ4RKTEfZZARcekcd2+TxSRp93vPxCRCe76BSKyRkTWu68XBu2TICIPichnIrJZRK6K5DkYY4zpKmJjHCISDzwALMB5zsEqEVmqqhuDNrsZOKSqk0XkOpxpFa7Fmd7hclXdLSLFwHKc2T3BmUK6SlVPdZ9tkB2pczDGjFwtLS1UVlbS2NjodSknLDMzk02bNvW7XVJSEoWFhfj9/pCOG8nB8blAhapuAxCRp4ArgeDguBJnumqAZ4Gfi4i4zx/oUA4ki0ii++yBbwNTAFS1HSdkjDEmrCorK0lPT2fChAk4c18OP7W1taSnp/e5japSXV1NZWUlEydODOm4kQyOsQQ9MwCn1XFWb9uoaquIHMZ5allwGFwFrFXVpqDpoP9FROYDW4FbenoYjogsAhYBFBQUUFZWNsjTiY66urphU+tA2bkNX7F8fr2dW2ZmJjk5OdTV1UW/qDBpa2ujtra23+0SEhKoqakJ+d/xkL4cV0SKcLqvLnFX+YBC4D1VvV1Ebgf+nWNPceukqg8BDwHMnj1bh8vldnZp4PAUy+cGsX1+vZ3bpk2byMjIiH5BYRRKi6NDUlISM2fODGnbSA6O7yLoYTM4f+Dv6m0bEfHhPO+42v1cCDwPfDPoUZ3VOE8l+6P7+Q/AmZEoXlV5/P0dvPjJ7v43NsaYESSSwbEKOEVEJroPm7kOWNptm6XATe77q4E3VVXdLqmXgMWq+m7Hxu6Mon8C5rurLqLrmEnYiAh/WL2TX/15WyQOb4wxfaqpqeEXv/jFgPf7yle+Qk1NTfgLChKx4HCfXXwLzhVRm4BnVLVcRO4WkSvczR4GckSkArgd6Lhk9xZgMnCXiKxzl3z3u+8BPxKRT3C6qO6I1DmUFAf4uPIwu2p6e1CbMcZERm/B0dra2sPWx7z88stkZWVFqCpHRMc4VPVlnEdZBq+7K+h9I3BND/st4diT2bp/twOIyuMuS4oC/HTZpyzfsJdvnxfa1QbGGBMOixcvZuvWrcyYMQO/309SUhKjRo1i8+bNfPbZZ3z1q19l586dNDY2cuutt7Jo0SLg2BRLdXV1LFy4kPPPP5/33nuPsWPH8sILL5CcnDzo2ob04LjXTs5L47SCdJZZcBgzov34T+Vs3H0krMecNiaDH15e1Ov399xzDxs2bGDdunWUlZVx6aWXsmHDhs5LZh955BGys7M5evQoc+bM4aqrriInJ6fLMbZu3crTTz/Nr371K772ta/x3HPP8Y1vdH+a8cDZlCP9KCkOsGrHQapqh+9NQMaY4W/u3Lld7rO4//77OeOMMzj77LPZuXMnW7ZsOW6f8ePHM2PGDABmzZrF9u3bw1KLtTj6UVIc4L/e2MJrG/dxw1njvS7HGOOBvloG0RI8PXpZWRmvv/46K1euJCUlhfnz5/d4h3tiYmLn+/j4eI4eDc94rbU4+jElkM6EnBSWbdjrdSnGmBEkPT2915v3Dh8+zKhRo0hJSWHz5s28//77Ua3NWhz9EBFKikfz67e3UdPQTFZKgtclGWNGgJycHObNm0dxcTHJyckUFBR0fldSUsKDDz7I1KlTOe200zj77LOjWpsFRwhKigM8+NZWXt9UxdWzCr0uxxgzQvz+97/vcX1iYiKvvPJKj991jGPk5ubywQcfdK7/7ne/G7a6rKsqBGcUZjImM4llG/Z4XYoxxnjOgiMEIsLC4gB/3nKAuqa+b74xxphYZ8ERopKiAM2t7azYXOV1KcYY4ykLjhDNnpBNblqCXV1ljBnxLDhCFB8nXFIUYMWnVTS2tHldjjHGeMaCYwBKigI0NLfx58/2e12KMcZ4xoJjAM6ZlENmst+6q4wxQ05aWlrUfsuCYwD88XFcPLWA1zfto7m13etyjDHGExYcA1RSHOBIYysrt1V7XYoxJoYtXryYBx54oPPzj370I5YsWcJFF13EmWeeyfTp03nhhRc8qc3uHB+gL52SS2pCPMs27OGCU/O8LscYEw2vLIa968N7zMB0KL2n16+vvfZabrvtNr7zne8A8Mwzz7B8+XL+7u/+joyMDA4cOMDZZ5/NFVdcgYiEt7Z+WItjgJL88Xx5Sj6vlu+jrV29LscYE6NmzpxJVVUVu3fv5uOPP2bUqFEEAgG+//3vc/rpp3PxxReza9cu9u3bF/XarMVxAkqKA7z4yR5WbT/I2Sfn9L+DMWZ466NlEEnXXHMNzz77LHv37uXaa6/liSeeYP/+/axZswa/38+ECRN6nE490qzFcQK+fFo+ib44u7rKGBNR1157LU899RTPPvss11xzDYcPHyY/Px+/38+KFSvYsWOHJ3VZcJyA1EQf55+ax7INe2m37ipjTIQUFRVRW1vL2LFjGT16NDfccAOrV69m+vTpPPbYY0yZMsWTuqyr6gSVFAV4beM+Pq6sYeZJo7wuxxgTo9avPzYon5uby8qVK3vcrq6uLlolWYvjRF08tQBfnFh3lTFmxLHgOEGZKX7OnZzLsvK9qFp3lTFm5LDgGISSogA7qhvYtKfn5wIbY4a3kfKXwoGepwXHIFxSVECcYE8GNCYGJSUlUV1dHfPhoapUV1eTlJQU8j42OD4IuWmJzJmQzbLyvdx+yWlel2OMCaPCwkIqKyvZv3/4zobd2NgYUiAkJSVRWFgY8nEtOAappDjAj/+0ka3765iUF73ZKY0xkeX3+5k4caLXZQxKWVkZM2fODPtxratqkEqKAwB2dZUxZsSw4Bik0ZnJzBiXZcFhjBkxLDjCoKQ4wPpdh9l5sMHrUowxJuIsOMKg1O2uWl5urQ5jTOyz4AiD8TmpTB2dYd1VxpgRwYIjTEqKAqz54hBVR6I/xbExxkRTRINDREpE5FMRqRCRxT18nygiT7vffyAiE9z1C0RkjYisd18vDNqnzD3mOnfJj+Q5hKp0egBVWL4x+g9VMcaYaIpYcIhIPPAAUApMA64XkWndNrsZOKSqk4H7gHvd9QeAy1V1OnAT8Hi3/W5Q1RnuUhWpcxiIU/LTODk31e4iN8bEvEi2OOYCFaq6TVWbgaeAK7ttcyXwqPv+WeAiERFV/UhVd7vry4FkEUmMYK2DJiKUFAd4f9tBDtU3e12OMcZETCTvHB8L7Az6XAmc1ds2qtoqIoeBHJwWR4ergLWq2hS07jci0gY8ByzRHiaTEZFFwCKAgoICysrKBnc2IchvaqOtXfn582/xpUL/CR2jrq4uKrV6wc5t+Irl87NzG7ghPeWIiBThdF9dErT6BlXdJSLpOMFxI/BY931V9SHgIYDZs2fr/PnzI16vqvKrTSv4vDWdf54/54SOUVZWRjRq9YKd2/AVy+dn5zZwkeyq2gWMC/pc6K7rcRsR8QGZQLX7uRB4Hvimqm7t2EFVd7mvtcDvcbrEhoSO7qp3thygtrHF63KMMSYiIhkcq4BTRGSiiCQA1wFLu22zFGfwG+Bq4E1VVRHJAl4CFqvqux0bi4hPRHLd937gMmBDBM9hwEqLAzS3tfPm5iExZm+MMWEXseBQ1VbgFmA5sAl4RlXLReRuEbnC3exhIEdEKoDbgY5Ldm8BJgN3dbvsNhFYLiKfAOtwWiy/itQ5nIgzTxpFXnqi3QxojIlZER3jUNWXgZe7rbsr6H0jcE0P+y0BlvRy2FnhrDHc4uKEhUUFPLdmF0eb20hOiPe6JGOMCSu7czwCSotHc7Sljbc+G74PgDHGmN5YcETA3InZZKX47WZAY0xMsuCIAH98HAumFvDGpiqaWtu8LscYY8LKgiNCSqcHqG1q5b2t1V6XYowxYWXBESHzJueSluhj2Xq7usoYE1ssOCIk0RfPhVPyeXXjXlrb2r0uxxhjwsaCI4JKiwMcamjhw+0HvS7FGGPCxoIjgi44LY8kf5zdDGiMiSkWHBGUkuDjglPzWF6+l/b24ybwNcaYYcmCI8JKi0ez70gTH+2s8boUY4wJCwuOCPvylHz88WI3AxpjYoYFR4RlJvuZNzmXZeV76eF5U8YYM+xYcERBaXGAnQePUr77iNelGGPMoFlwRMHFUwuIE+zqKmNMTLDgiIKctETOmpjDsnILDmPM8GfBESWl0wNUVNVRUVXrdSnGGDMoFhxRcsm0AACv2NxVxphhzoIjSgKZSZx5UpZ1Vxljhj0LjigqLR5N+e4jfFHd4HUpxhhzwiw4oqik2OmuWlZuNwMaY4YvC44oGpedQtGYDLss1xgzrFlwRFlpcYC1X9Sw93Cj16UYY8wJseCIso7uquU2SG6MGaYsOKJscn46k/PTrLvKGDNsWXB4oLQ4wAefV1Nd1+R1KcYYM2D9BoeI5InI90XkIRF5pGOJRnGxamFRgHaF1zbu87oUY4wZMF8I27wAvA28DrRFtpyRoWhMBuOyk1lWvpfr5p7kdTnGGDMgoQRHiqp+L+KVjCAiQklRgN++t53DR1vITPZ7XZIxxoQslDGOF0XkKxGvZIQpKR5NS5vy5mbrrjLGDC+hBMetOOHRKCK17mJPJBqkmeOyKMhItKurjDHDTr/Boarpqhqnqknu+3RVzYhGcbEsLk5YWBTgrc/209Dc6nU5xhgTspAuxxWRK0Tk393lskgXNVKUFAdobGnnrU/3e12KMcaELJTLce/B6a7a6C63isi/RrqwkWDuhGyyUxN4xbqrjDHDSCgtjq8AC1T1EVV9BCgBLg3l4CJSIiKfikiFiCzu4ftEEXna/f4DEZngrl8gImtEZL37emEP+y4VkQ2h1DFU+eLjWDC1gDc3V9HUalc6G2OGh1DvHM8Kep8Zyg4iEg88AJQC04DrRWRat81uBg6p6mTgPuBed/0B4HJVnQ7cBDze7dh/CdSFWPuQVjI9QF1TK+9WHPC6FGOMCUkowfGvwEci8lsReRRYA/wkhP3mAhWquk1Vm4GngCu7bXMl8Kj7/lngIhERVf1IVXe768uBZBFJBBCRNOB2YEkINQx58yblkp7os0fKGmOGjX5vAFTVJ0WkDJjjrvqeqobyp9xYYGfQ50rgrN62UdVWETkM5OC0ODpcBaxV1Y6Jnf4F+BkQE4/RS/DFcdHUfF7btI+WtnavyzHGmH71GhwiMkVVN4vIme6qSvd1jIiMUdW1kS5ORIpwuq8ucT/PACap6t93jIf0se8iYBFAQUEBZWVlEa11MMZJKzUNLfzqv1dwUuLRIV3rYNTV1dm5DVOxfH52bgPXV4vjdpw/eH/Ww3cKHDdg3c0uYFzQ50J3XU/bVIqID2f8pBpARAqB54FvqupWd/tzgNkist2tPV9EylR1/nEFqj4EPAQwe/ZsnT//uE2GjLOa2/j1htfYHZ/PtLRqhnKtg1FWVmbnNkzF8vnZuQ1cr8Ghqovct6Wq2uVxdSKSFMKxVwGniMhEnIC4Dvh6t22W4gx+rwSuBt5UVRWRLOAlYLGqvhtU0y+BX7o1TABe7Ck0hpvkhHjmn5bH8vJ9XHhuvNflGGNMn0IZHH8vxHVdqGorcAuwHNgEPKOq5SJyt4hc4W72MJAjIhU4LZyOS3ZvASYDd4nIOnfJD6HWYaukOMD+2ia21tg4hzFmaOtrjCOAM3idLCIzAXG/ygBSQjm4qr4MvNxt3V1B7xuBa3rYbwn9XDWlqtuB4lDqGA4unJJPQnwcq/e28jdeF2OMMX3oa4xjIfBXOGMTP+NYcBwBvh/Zskae9CQ/552Sy+rt+1FVRKT/nYwxxgN9jXE8CjwqIlep6nNRrGnEKikO8ObmKjbsOsL0wpDuszTGmKgLZYxjljtYDYCIjBKRmLj5bqhZMLWAOIFXNuzxuhRjjOlVKMFRqqo1HR9U9RDO/FUmzEalJjAlO45lG/aiql6XY4wxPQolOOI7pvsAEJFkILGP7c0gzC7wse1APVuqYmIqLmNMDAolOJ4A3hCRm0XkZuA1js0vZcLszPx4RLC5q4wxQ1YoTwC8F2dSw6nu8i+q+tNIFzZSZSXFMeukUTbOYYwZsvqd5BBAVV8BXolwLcZVUhxgyUub2H6gngm5qV6XY4wxXYTyBMC/FJEtInJYRI6ISK2IHIlGcSNVSXEAgGXl1l1ljBl6Qhnj+ClwhapmqmqGqqarakakCxvJCkelMH1spj1S1hgzJIUSHPtUdVPEKzFdlBQH+HhnDbtrjnpdijHGdBFKcKx2nwt+vdtt9Zfuo1tNBJW63VXLrbvKGDPEhBIcGThP27sEuNxdLotkUQZOzkvj1II0lll3lTFmiAnl0bHfikYh5nglxaP5+ZtbOFDXRG6a3XNpjBka+g0OEfkNzhP/ulDVb0ekItOptDjA/W9s4dXyfXz9rJO8LscYY4DQuqpexHka30vAGzhdVzYfRhRMCaQzPifFLss1xgwpoXRVdZlSXUSeBN6JWEWmk4hQUhzg4bc/53BDC5kpfq9LMsaYkFoc3Z0CxPRjXIeSkqIAre3K65v2eV2KMcYAod05XuveMX7EvWP8T8D3Il+aATijMIvRmUnWXWWMGTL6eub4PFV9F8hznw1uPBAXJywsCvDkh19Q39RKamJI04sZY0zE9NXiuN99fS8ahZjelRQHaGptZ8WnVV6XYowxfQ6Ot4jIQ0ChiNzf/UtV/bvIlWWCzZmQTW5aAss27OWy08d4XY4xZoTrKzguAy4GFgJrolOO6Ul8nLBgWoCl63bR2NJGkj/e65KMMSNYr8GhqgeAp0Rkk6p+HMWaTA9Kip1xjre3HGDBtAKvyzHGjGChPAHQQmMIOOfkHDKSfDZ3lTHGcydyH4fxQIIvjounFfD6pn20tLV7XY4xZgSz4BhGSooCHD7awsqt1V6XYowZwUK5AfBWEckQx8MislZELolGcaar80/NIyUh3m4GNMZ4KpQWx7dV9QjO8zhGATcC90S0KtOjJH88X56Sz6vle2lrP27CYmOMiYpQgkPc168Aj6tqedA6E2UlRQEO1DWzevtBr0sxxoxQoQTHGhF5FSc4lotIOmCjsx758pR8Enxx1l1ljPFMKMFxM7AYmKOqDYAfsKcCeiQt0cf5p+SxfMNeVK27yhgTfaEExznAp6paIyLfAH4AHI5sWaYvJcUBdh9u5ONK+9dgjIm+UILjl0CDiJwB3AFsBR4L5eAiUiIin4pIhYgs7uH7RBF52v3+AxGZ4K5fICJrRGS9+3ph0D7LRORjESkXkQdFZMTNv7FgagG+OLGbAY0xngglOFrV6RO5Evi5qj4ApPe3k/sH+gNAKTANuF5EpnXb7GbgkKpOBu4D7nXXHwAuV9XpwE3A40H7fE1VzwCKgTzgmhDOIaZkpvg5Z1IOyzbsse4qY0zUhRIctSJyJ85luC+JSBzOOEd/5gIVqrpNVZuBp3DCJ9iVwKPu+2eBi0REVPUjVd3tri8HkkUkEcC9NBicebYSgBH5J2dJcYDt1Q18uq/W61KMMSNMKMFxLdCEcz/HXqAQ+LcQ9hsL7Az6XOmu63EbVW3FGTvJ6bbNVcBaVW3qWCEiy4EqoBYncEacS6YFEIFX1lt3lTEmuvp9nJyq7hWRJ4A5InIZ8KGqhjTGMVgiUoTTfdXlTnVVXSgiScATwIXAaz3suwhYBFBQUEBZWVnE6w2Hurq6kGs9NSuO5z6oYKZ/d/8bDwEDObfhJpbPDWL7/OzcToCq9rkAXwN24HQpPQZ8Dlwdwn7nAMuDPt8J3Nltm+XAOe57H87YhrifC4HPgHl9/MY3ccZd+qxl1qxZOlysWLEi5G0ffnubjv/ei7q1qjZyBYXRQM5tuInlc1ON7fOzc+sdsFp7+DM1lK6qf8K5h+MmVf0mztjFP4ew3yrgFBGZKCIJwHXA0m7bLMUZ/Aa4GnhTVVVEsoCXgMXqPPccABFJE5HR7nsfcCmwOYRaYlJJcQDAbgY0xkRVKMERp6rBD7uuDmU/dcYsbsFpVWwCnlHVchG5W0SucDd7GMgRkQrgdpwbDXH3mwzcJSLr3CUfSAWWisgnwDqccY4HQziHmDQmK5kzxmXZZbnGmKjqd4wDWOYORj/pfr4WeDmUg6vqy923VdW7gt430sPltKq6BFjSy2HnhPLbI0VJUYB7l21mV81RxmYle12OMWYECKXl8A/AQ8Dp7vKQqn4v0oWZ0JR2dFdZq8MYEyWhtDhQ1eeA5yJcizkBE3JTmRJIZ9mGPdx83kSvyzHGjAC9tjhEpFZEjvSw1IrIkd72M9FXUhxg9Y5DVNU2el2KMWYE6DU4VDVdVTN6WNJVNSOaRZq+lRaPRhVeLd/ndSnGmBHAnjkeA04tSOPk3FQb5zDGRIUFRwwQERYWB1i5rZqahmavyzHGxDgLjhhRWhygrV15baN1VxljIsuCI0ZMH5vJ2Kxk664yxkScBUeMEBEWFgV4e8sB6ppavS7HGBPDLDhiSOn0AM1t7by5uar/jY0x5gRZcMSQM08aRW5aIss27PG6FGNMDLPgiCHxccLCogJWbN5PY0ub1+UYY2KUBUeMKS0ezdGWNt76bL/XpRhjYpQFR4w56+RsMpP9dnWVMSZiLDhijD8+jgXTCnh90z6aW9u9LscYE4MsOGJQaXGA2sZW3tt6wOtSjDExyIIjBs2bnEtqQjzL7ZGyxpgIsOCIQUn+eC6cWsCr5ftoa1evyzHGxBgLjhhVWhygur6ZDz8/6HUpxpgYY8ERoy44NY9EX5x1Vxljws6CI0alJvq44NQ8lm3YS7t1VxljwsiCI4aVTg+w90gj6yprvC7FGBNDLDhi2IVTCvDHC8vtZkBjTBhZcMSwzGQ/507K5ZUNe1G17ipjTHhYcMS40uIAXxxsYOOeI16XYoyJERYcMW7BtALiBOuuMsaEjQVHjMtJS2TuxGxeseAwxoSJBccIUFo8mi1VdVRU1XldijEmBlhwjAALiwIAdjOgMSYsLDhGgEBmEjNPyuIVe6SsMSYMLDhGiNLiABt2HWHnwQavSzHGDHMWHCNESdFowLqrjDGDZ8ExQpyUk8K00Rl2dZUxZtAiGhwiUiIin4pIhYgs7uH7RBF52v3+AxGZ4K5fICJrRGS9+3qhuz5FRF4Skc0iUi4i90Sy/lhTWhxgzY5D7DvS6HUpxphhLGLBISLxwANAKTANuF5EpnXb7GbgkKpOBu4D7nXXHwAuV9XpwE3A40H7/LuqTgFmAvNEpDRS5xBVLY3w3N8w6uBaaI/Ms8JLip2rq1617ipjzCBEssUxF6hQ1W2q2gw8BVzZbZsrgUfd988CF4mIqOpHqrrbXV8OJItIoqo2qOoKAPeYa4HCCJ5D9Bz4DLa+yRmf/Bj+75nw7n9BfXVYf+KUgnQm5aVad5UxZlAiGRxjgZ1BnyvddT1uo6qtwGEgp9s2VwFrVbUpeKWIZAGXA2+Er2QPjT4dbt/Ixql3QPpoeO0u+I+p8MdF8MUHEKZJCkuLR/PB5wc5WN8cluMZY0Yen9cF9EVEinC6ry7ptt4HPAncr6rbetl3EbAIoKCggLKyssgWGyZ1qWdSVXA+qfk7GLN7GQXlS/F98jR1qRPYPaaEfQUX0OZLOeHj5za20dau/Pz5tzi/0B/GyvtXV1c3bP49DFQsnxvE9vnZuZ0AVY3IApwDLA/6fCdwZ7dtlgPnuO99OGMb4n4uBD4D5vVw7EdwQiOkWmbNmqXDxYoVK7quaKxVXfWI6i/nqf4wQ/UnY1T/9Peqezec0PHb29t13j1v6F898sHgix2g484thsTyuanG9vnZufUOWK09/Jkaya6qVcApIjJRRBKA64Cl3bZZijP4DXA18KaqqtsN9RKwWFXfDd5BRJYAmcBtEax96EhMg9nfgv/xNtz8Oky9HD76HfzyXHh4IXzyjDOwHiIRobQ4wLsV1RxpbIlg4caYWBWx4FBnzOIWnFbFJuAZVS0XkbtF5Ap3s4eBHBGpAG4HOi7ZvQWYDNwlIuvcJV9ECoF/wrlKa627/q8jdQ5DigiMmwN/8SDcsRkuWQL1VfDHv4H7psGr/wwHe+y1O05JcYDmtnZWbK6KcNHGmFgU0TEOVX0ZeLnburuC3jcC1/Sw3xJgSS+HlXDWOCylZMO5fwtnfwc+L4NVD8PKB+C9+2HSRTDnZjhlIcT3/K935rhR5Kcn8vL6PVw5o/v1CsYY07chPThu+hEXB5MudJYju2HNo7D2UXjq65BRCLNugjO/CemBbrsJJcUBHlu5g5l3v8qkvDQm56d1vk7OT2NsVjJxcZbRxpjjWXDEiowx8OU74fzvwqevwOqHYcVP4K17YcqlMPtmmHi+0+UF3HbxqYzPSaWiqo6tVXW8tnEfT9Ufu3o60RfHyXlpTMpL7QyTSXlpTMxNJckf79VZGmOGAAuOWBPvh2lXOEv1Vlj9iDOYvvEFyDkFZn8bZlxPduoobj5vYpddD9U3s3W/88Cniqo6tu6v4+PKGl5av6fzNpI4gXHZKUGtFDdY8tLJTInu5b3GGG9YcMSynEmw8Cdw4Q+g/HknRJbfCW/cDcVXwZxvw9hZnZuPSk1gdmo2sydkdzlMY0sb2/bXU7HfaZ10vL5TcYDm1mPTo+SmJTApL41J+WlM7giW/DTGZCYhYt1exsQKC46RwJ8MM77uLHs+dgLkkz/Aut/B6BnOYHrx1ZDQ842FSf54po3JYNqYjC7r29qVykMN3Vop9bz0yR4OHz12qW9KQjyT8tJIa29kQ/uWztbK+JxUEnw2QbMxw40Fx0gz+gy4/L9gwd3OPSCrfg1L/xaW/wBmXO90ZeWdFtKh4uOE8TmpjM9J5cIpBZ3rVZXq+uYuXV4VVXVs3NnOylc/67p/dgqTug3MT8pLJT3Jur2MGaosOEaqpEyY+zcw569hx3vOYPqqh+GDB2HCl5wAmXIZ+BIGfGgRITctkdy0RM4++djUY2VlZcw55zy27a/vDJOO17JPq2hpOzYfV0FGYrcwcV7z0xOt28sYj1lwjHQiMGGes9RVwUePw+rfwrPfgtR853LeWX8FWePC8nOpiT6mF2YyvTCzy/qWtnZ2Hmzo7O6qcMdSnl+7i9qm1s7t0hN9nOyOoUzKT+0cSzkpOwVfvHV7GRMNFhzmmLR8+NIdMO82qHjdaYG8/TN45z+cGwrn3OzcYBgX/j+g/fHO5b8n56V1Wa+qVNU2dQ7Kd7RS3qnYz3NrK4P2FybkpB672is/lYL0JLLTEshJTWRUit+CxRzT3gb7NsDODxlbuRm2tEDOZMg6CeLscvP+WHCY48XFw6kLneXQDuemwrWPwWevQNZ4Z+6smTdCam7ESxERCjKSKMhI4tzJXX+vtrGls3XS0eX1WVUtr23aR1u7djsOZCX7yUlLJDs1gdy0BLJTnVDJccMleH1WSgLxdgNk7Giqg12r4Yv3naVyFTTXAXAKQMWvne3iE2DURCdEcia5r+6Slt95H9RIZ8Fh+jZqPFx0F1ywGDb/CVY9Aq//CFb8H5h2pXNj4Ulne/I/VHqSnxnjspgxLqvL+ubWdr442MCBuiaq65o5WN/EgbpmDtY3U13vrPtsXx3VdU3UHG3p8VEncQKjUhLI6QiYtERyUxPI7gyahC4h1B6m56WYMDmyB3a+7zzL5ouVsHc9aBsgUFAEp18LJ50DJ53Fux+uZd5p+VBd4S5bndeK16At6Lk1CenHh0nH56SMXkuJRRYcJjS+BOfej+KroGqzc0nvx0/C+j9A/jRnMP30a4fE/0AJvrjOQfX+tLa1c6ihxQmVuiaq3deD9c0cqG/mYJ0TNpt2H6G6vrnLZcbB4gRy3nvdDRQ3YDpbMMeHTUaSzwb5w6W9HQ586gRER1DU7HC+8yVD4Ww47++doCicDclZXXZvSdgG4891li7HbYPDO7uGSXUFVH4IG54Dgv6ykJoPuaccHyyjJoAvMZJn7wkLDjNw+VPgKz+Fi3/oBMeqh+Hl78JrP4TTv+aMhQSme11lSHzxceSlJ5KXngik97t9S1s7h+qbj2vBrN24hfTc/M716ytrqK5r7jKwH8wfL926yhK6daN1XZ+aEG9B06GlEXavPRYUOz+Axhrnu9Q8pwU8d5ETFIHpJ3RlIOB02Y6a4CyTLz6+hkOfB7VS3HD59BWo339sO4lzxk16aqVkFEZkvDAaLDjMiUtIda64OvMm2LXGCZCPn4Q1v4HCuU6ATPsq+JO8rjRs/PFx5GckkZ/R9ZxObt3B/PmnH7d9U2ub25ppprre6TbreN/ZsqlrZkd1A9V1TdQ3t/X4uwm+OKerzB2PyUj2k5oQT3JCPKkJPpIT4kkJep+aGE+y30dqorM+JcHX+TrsbrqsP+CEQ0dQ7P4I2t2WX+6pzvQ64852AiP75Oh0m/qTIH+qs3R3tAYObnWC5MCWY8GyYyW01B/bzpfk1Htc99dkSMkZ0uMpFhxm8EScLoDC2c4UJx8/6XRlPf8/YNmdMPMGmPUtr6v0RKIvntGZyYzOTA5p+8aWts5QqXYDp3vYVNc388XBBhqaW2loaqOhpe24iwH64ouTrmGSGE+K3+e8dguZFDeQKne2UPPRrmPrE7sFVYKPJH/c4FtFqs5zZb5YeSwoqrc438UnwJiZcM7/doJi3FmQmtP38byQnOVM5RM0nQ/gnFvt3uNbKVWbnZZKe1DrNCnTmVuueyslZ5LzFzaPWXCY8ErJhnO+A2f/b/j8LfdZIb+A9/4vs9ImQe2XnGlOxsxwxkZisP93MJL88YzNSmZsVmhBA84ly02t7RxtbqO+udV9besSLEebW6lvauNoSxv1Ta00dHzf3Nb5/mB9M5WH2mhoaqWhpY2Gpjaa247NRUb5uj7rEIEUfzzJCT63xeOGS6KPZL/7mhDvtpR8pCbEk+ZrZ/TRLYw+/BG5h9aRdWAN/sZqANqTstDCs4ibcQNy0tlOaAzn1qsIZIx2lolf6vpdW6szLhM8llJdAdvfgU+e6rpt+pieWymjxjuTnEaBBYeJDBE4eb6zHNkDH/2O1o+WOpMtrvmts02cHwqmuUEy08LkBIkISf54kvzxjEo9wf78XrS0tdPQ3Mabb73N6bPmctQNms6Aamp1w8gNp6Ag6nita2ql6kgTDS2txDUe4bSWTZyumzgj7jNmSAVJ4nQ77WjP5886jdXtp7Kq/TS2No5Ba+LwbRTSkmpJTXiPtEQfaUk+UhN9pCXGk5bovE93X1MTfaQn+UhNCHqf6HP2SwxTqygS4n1uGEwCLun6XXOD0wrrftXXxv+Go4eObSfumExQKyXrUD20nnvi4zy9sOAwkZcxGi74Bz7WOcy/4AI4tB32rHP6qnevc/4HWPuos62FyZDij48jMzmOUUlxTMrr/yq1LlSh5ouu4xMNGyFeUYmnvWA6TWO+zf6CWdTknMkRfw75zW1c0NTKrCYnlOrcpb6plbpG931zK4cbmtl1yGlFdawL5Yro+DhxWjpuyKQl+WiuP8rTlWu6BExnMLkhFBxYHSGVEq0LFhJSIFDsLN01HOzW9eUGy+d/htajzAC49K8BCw4znIlA9kRnKfoLZ51qUJiscwKl1zCZ4QSKhcnQ09bq3o0dFBS1u53vEtJh3Bwo+iqMOwspnE18QiopQAqQN8ifbm9XGlqCwqax9bjgqXVf65vaqHW/r29uZddhqKiq67JtKENGIpCW0NHSiSctyd+lFdQ1hI69P70wkzED6IrsU0o2pMyFcXO7/wOB2t18tOIFZvYy6/VgWHAY74UjTEbPcG7ssjCJni53Y6+EytWdd2OTMRbGn+NcEjvuLOffTQSn8oiLk84/nAv637yLsrIy5s+/oPOzqnK0xW3JNLV1tnQ6Wz7d3zc6AdQRRgdqmzu3qWtqPe7ChfuuPYO/mFkYhrPuQ1wcZBZyOKsoIoe34DBDUyhhsmed82RDC5Po6Lwb21263I1dDGdcdywowjQpphdExL2izBfKrT196rhwobPV09g6oAsfhioLDjN8WJhETmuTc+Na/X7nvon6/c5syfX7oXYPZ1W8A2X7nG077sb+0u3OvROFc5zLR81xgi9cyE2Lnf/mLDjM8GZh0jNV54qbjhCorwp63xEKB44FRdPhno/jS4a0fOrSTib5gludoAicHrXLPs3QZMFhYk9vYVKz49iVXD2FSf7UY1dyDcUwaWk8vlXQ01K3HxoOdL2hrJM4dyWn5jmzG4+Zcex9ap4z51Lw54RUEKG8rIz558yP8gmbocqCw4wMIsfmHRpwmMxwAiXcYdLe7syx1GMroPtyAJqO9HwcXzKkuX/oZxQ6dabmOdOAdwmFPCc07HkTZpAsOMzI1WeYrHMCZc862LjUeR4J9B4mHVqO9twqqOsWAh1dR9rD3FQS161VMNNtCQQFQFp+11aBMVFkwWFMsC5h8lVnXQhhclbCKHjvKDTX9nxcf+qxP+izxsHYmcdCoPuSkm2tAjOkWXAY05/+wmTPOo58tpbkidPcLqPuS661CkxMseAw5kR0C5NNvjIK5s/3uChjomOYTcxvjDHGaxYcxhhjBsSCwxhjzIBYcBhjjBmQiAaHiJSIyKciUiEii3v4PlFEnna//0BEJrjrF4jIGhFZ775eGLTPT0Rkp4jURbJ2Y4wxPYtYcIhIPPAAUApMA64XkWndNrsZOKSqk4H7gHvd9QeAy1V1OnAT8HjQPn8Cuk0+b4wxJloi2eKYC1So6jZVbQaeAq7sts2VgDu/A88CF4mIqOpHquo+AYZyIFlEEgFU9X1V3RPBuo0xxvQhksExFtgZ9LnSXdfjNqraChwGcrptcxWwVlWbIlSnMcaYARjSNwCKSBFO99Ul/W3bw76LgEXuxzoR+TSctUVQLk5XXSyycxu+Yvn87Nx6N76nlZEMjl1A8GPACt11PW1TKSI+IBOoBhCRQuB54JuqunWgP66qDwEPnUDdnhKR1ao62+s6IsHObfiK5fOzcxu4SHZVrQJOEZGJIpIAXAcs7bbNUpzBb4CrgTdVVUUkC3gJWKyq70awRmOMMQMUseBwxyxuAZYDm4BnVLVcRO4WkSvczR4GckSkArgd6Lhk9xZgMnCXiKxzl3wAEfmpiFQCKSJSKSI/itQ5GGOMOZ6oqtc1mCAissjtZos5dm7DVyyfn53bCRzXgsMYY8xA2JQjxhhjBsSCwxhjzIBYcAwRIvKIiFSJyAavawknERknIitEZKOIlIvIrV7XFE4ikiQiH4rIx+75/djrmsJNROJF5CMRedHrWsJNRLa7c+KtE5HVXtcTTiKSJSLPishmEdkkIueE7dg2xjE0iMj5QB3wmKoWe11PuIjIaGC0qq4VkXRgDfBVVd3ocWlhISICpKpqnYj4gXeAW1X1fY9LCxsRuR2YDWSo6mVe1xNOIrIdmK2qMXcDoIg8Crytqr92b4lIUdWacBzbWhxDhKr+GTjodR3hpqp7VHWt+74W59Ls7lPPDFvq6Jip2e8uMfO3MfdG3EuBX3tdiwmdiGQC5+Pc8oCqNocrNMCCw0SRO23+TOADj0sJK7crZx1QBbymqrF0fv8J/CPQ7nEdkaLAq+7jGxb1u/XwMRHYD/zG7Wb8tYikhuvgFhwmKkQkDXgOuE1Vj3hdTzipapuqzsCZVmeuiMREV6OIXAZUqeoar2uJoPNU9Uycxz98x+0yjgU+4Ezgl6o6E6jn2A3Wg2bBYSLO7ft/DnhCVf/odT2R4nYFrABKPC4lXOYBV7jjAE8BF4rI77wtKbxUdZf7WoUzN16sPOunEqgMav0+ixMkYWHBYSLKHTx+GNikqv/hdT3hJiJ57txqiEgysADY7GlRYaKqd6pqoapOwJlr7k1V/YbHZYWNiKS6F2zgduNcAsTEVY2quhfYKSKnuasuAsJ2QcqQnlZ9JBGRJ4H5QK47F9cPVfVhb6sKi3nAjcB6dxwA4Puq+rJ3JYXVaOBR94mXcThzssXcZasxqgB43vm7DT7g96q6zNuSwupvgSfcK6q2Ad8K14HtclxjjDEDYl1VxhhjBsSCwxhjzIBYcBhjjBkQCw5jjDEDYsFhjDFmQCw4jBnCRGR+LM5Ka4Y3Cw5jjDEDYsFhTBiIyDfc53KsE5H/5058WCci97nP6XhDRPLcbWeIyPsi8omIPC8io9z1k0XkdffZHmtFZJJ7+LSg5yo84d6Nb4xnLDiMGSQRmQpcC8xzJztsA24AUoHVqloEvAX80N3lMeB7qno6sD5o/RPAA6p6BnAusMddPxO4DZgGnIxzN74xnrEpR4wZvIuAWcAqtzGQjDPFejvwtLvN74A/us9JyFLVt9z1jwJ/cOdMGquqzwOoaiOAe7wPVbXS/bwOmIDzwChjPGHBYczgCfCoqt7ZZaXIP3fb7kTn92kKet+G/X9rPGZdVcYM3hvA1SKSDyAi2SIyHuf/r6vdbb4OvKOqh4FDIvIld/2NwFvu0xErReSr7jESRSQlmidhTKjsby7GDJKqbhSRH+A8SS4OaAG+g/PwnLnud1U44yAANwEPusEQPGvpjcD/E5G73WNcE8XTMCZkNjuuMREiInWqmuZ1HcaEm3VVGWOMGRBrcRhjjBkQa3EYY4wZEAsOY4wxA2LBYYwxZkAsOIwxxgyIBYcxxpgB+f+AhoZOxJyX4AAAAABJRU5ErkJggg==\n",960 "text/plain": [961 "<Figure size 432x288 with 1 Axes>"962 ]963 },964 "metadata": {965 "needs_background": "light"966 },967 "output_type": "display_data"968 },969 {970 "name": "stdout",971 "output_type": "stream",972 "text": [973 "Epoch 7\n",974 "-------------------------------\n"975 ]976 },977 {978 "data": {979 "text/plain": [980 "Learning..."981 ]982 },983 "metadata": {},984 "output_type": "display_data"985 },986 {987 "data": {988 "application/vnd.jupyter.widget-view+json": {989 "model_id": "59aed5898485404a8a18eb9f6300eadc",990 "version_major": 2,991 "version_minor": 0992 },993 "text/plain": [994 "HBox(children=(HTML(value='Batch #'), FloatProgress(value=0.0, max=145.0), HTML(value='')))"995 ]996 },997 "metadata": {},998 "output_type": "display_data"999 },1000 {1001 "data": {1002 "text/plain": [1003 "'[0/36900] Loss: 0.021270'"1004 ]1005 },1006 "metadata": {},1007 "output_type": "display_data"1008 },1009 {1010 "data": {1011 "text/plain": [1012 "'[1845/36900] Loss: 0.021709'"1013 ]1014 },1015 "metadata": {},1016 "output_type": "display_data"1017 },1018 {1019 "data": {1020 "text/plain": [1021 "'[3690/36900] Loss: 0.021914'"1022 ]1023 },1024 "metadata": {},1025 "output_type": "display_data"1026 },1027 {1028 "data": {1029 "text/plain": [1030 "'[5535/36900] Loss: 0.021512'"1031 ]1032 },1033 "metadata": {},1034 "output_type": "display_data"1035 },1036 {1037 "data": {1038 "text/plain": [1039 "'[7380/36900] Loss: 0.020658'"1040 ]1041 },1042 "metadata": {},1043 "output_type": "display_data"1044 },1045 {1046 "data": {1047 "text/plain": [1048 "'[9225/36900] Loss: 0.021273'"1049 ]1050 },1051 "metadata": {},1052 "output_type": "display_data"1053 },1054 {1055 "data": {1056 "text/plain": [1057 "'[11070/36900] Loss: 0.021641'"1058 ]1059 },1060 "metadata": {},1061 "output_type": "display_data"1062 },1063 {1064 "data": {1065 "text/plain": [1066 "'[12915/36900] Loss: 0.020861'"1067 ]1068 },1069 "metadata": {},1070 "output_type": "display_data"1071 },1072 {1073 "data": {1074 "text/plain": [1075 "'[14759/36900] Loss: 0.021167'"1076 ]1077 },1078 "metadata": {},1079 "output_type": "display_data"1080 },1081 {1082 "data": {1083 "text/plain": [1084 "'[16605/36900] Loss: 0.021825'"1085 ]1086 },1087 "metadata": {},1088 "output_type": "display_data"1089 },1090 {1091 "data": {1092 "text/plain": [1093 "'[18449/36900] Loss: 0.021072'"1094 ]1095 },1096 "metadata": {},1097 "output_type": "display_data"1098 },1099 {1100 "data": {1101 "text/plain": [1102 "'[20294/36900] Loss: 0.021985'"1103 ]1104 },1105 "metadata": {},1106 "output_type": "display_data"1107 },1108 {1109 "data": {1110 "text/plain": [1111 "'[22140/36900] Loss: 0.020836'"1112 ]1113 },1114 "metadata": {},1115 "output_type": "display_data"1116 },1117 {1118 "data": {1119 "text/plain": [1120 "'[23985/36900] Loss: 0.022309'"1121 ]1122 },1123 "metadata": {},1124 "output_type": "display_data"1125 },1126 {1127 "data": {1128 "text/plain": [1129 "'[25830/36900] Loss: 0.021078'"1130 ]1131 },1132 "metadata": {},1133 "output_type": "display_data"1134 },1135 {1136 "data": {1137 "text/plain": [1138 "'[27675/36900] Loss: 0.021097'"1139 ]1140 },1141 "metadata": {},1142 "output_type": "display_data"1143 },1144 {1145 "ename": "KeyboardInterrupt",1146 "evalue": "",1147 "output_type": "error",1148 "traceback": [1149 "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",1150 "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",1151 "\u001b[0;32m<ipython-input-4-5ec6961c37c7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Epoch {epoch+1}\\n-------------------------------\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mtrain_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msci_classifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msci_train_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 18\u001b[0m \u001b[0mtest_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msci_classifier\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msci_test_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhistory_loss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",1152 "\u001b[0;32m<ipython-input-21-09a3ee852cb4>\u001b[0m in \u001b[0;36mtrain_loop\u001b[0;34m(model, dataloader, loss_fn, optimizer, step, history_loss)\u001b[0m\n",1153 "\u001b[0;31mKeyboardInterrupt\u001b[0m: "1154 ]1155 }1156 ],1157 "source": [1158 "#!g1.1\n",1159 "from transformers import AutoModelForSequenceClassification\n",1160 "sci_classifier = AutoModelForSequenceClassification.from_pretrained(sci_model_name, num_labels=num_tags).to('cuda')\n",1161 "for param in sci_classifier.base_model.parameters():\n",1162 " param.requires_grad = False\n",1163 "\n",1164 "sci_classifier.train()\n",1165 "loss_fn = F.binary_cross_entropy\n",1166 "optimizer = torch.optim.Adam(sci_classifier.parameters(), lr=1e-4)\n",1167 "epochs = 30\n",1168 "\n",1169 "train_loss = []\n",1170 "val_loss = []\n",1171 " \n",1172 "for epoch in range(epochs):\n",1173 " print(f\"Epoch {epoch+1}\\n-------------------------------\")\n",1174 " \n",1175 " train_loop(sci_classifier, sci_train_dl, loss_fn, optimizer, history_loss=train_loss)\n",1176 " test_loop(sci_classifier, sci_test_dl, loss_fn, history_loss=val_loss)\n",1177 " \n",1178 " clear_output()\n",1179 " plot_learning_process(train_loss, val_loss)"1180 ]1181 },1182 {1183 "cell_type": "markdown",1184 "metadata": {1185 "cellId": "4tyrvht7r3rk4dny2oyk5"1186 },1187 "source": [1188 "Видим, что разницы в результатах не заметно, поэтому остановим обучение."1189 ]1190 },1191 {1192 "cell_type": "markdown",1193 "metadata": {1194 "cellId": "cmyx82lx7eixgamzj6jbza"1195 },1196 "source": [1197 "Теперь попробуем датасет побольше (существенно побольше). Он взят [отсюда](https://www.kaggle.com/datasets/Cornell-University/arxiv?select=arxiv-metadata-oai-snapshot.json). Мы используем только часть датасета (100'000 статей)."1198 ]1199 },1200 {