hschumann2/TempleOS-Source-Code

1 2/*3This file is a stand-alone program4which will regenerate processed dictionary5files from a raw Project Gutenberg6dictionary file.7 8/files/DICTIONARY.DD9 10See ::/Doc/Credits.DD.11*/12 13U0 ACDPreprocess(U8 *in_name,U8 *out_name)14{/*15<cr><nl>--> <nl>16$       --> $$17\'89    --> .18*/19  I64 ch,i;20  U8 *src,*dst;21  CDoc *doc;22  CDocEntry *doc_e;23  if (doc=DocRead(in_name,DOCF_PLAIN_TEXT_TABS|DOCF_DBL_DOLLARS)) {24    doc_e=doc->head.next;25    while (doc_e!=doc) {26      if (doc_e->type_u8==DOCT_TEXT) {27        src=dst=doc_e->tag;28        while (ch=*src++) {29          if (ch=='\\' && *src=='\'') {30            src++;31            i=0;32            ch=ToUpper(*src++);33            if ('0'<=ch<='9')34              i+=ch-'0';35            else if ('A'<=ch<='F')36              i+=ch-'A'+10;37            i<<=4;38            ch=ToUpper(*src++);39            if ('0'<=ch<='9')40              i+=ch-'0';41            else if ('A'<=ch<='F')42              i+=ch-'A'+10;43            *dst++=i;44          } else45            *dst++=ch;46        }47        *dst=0;48      }49      doc_e=doc_e->next;50    }51    StrCpy(doc->filename.name,out_name);52    DocWrite(doc);53    DocDel(doc);54  }55}56 57I64 ACDNextCmd(U8 **_ptr)58{59  U8 *ptr=*_ptr,*ptr2;60  I64 ch,res=-1;61  do {62    do {63      if (!(ch=*ptr++)) goto ncmd_done;64    } while (ch!='<');65 66    ptr2=ptr;67    do {68      if (!(ch=*ptr2++)) goto ncmd_done;69    } while (ch!='>');70    *--ptr2=0;71    res=LstMatch(ptr,"h1\0/h1\0def\0/def\0hw\0/hw\0tt\0/tt\0"72          "ety\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\[email protected]\0"73          "@cref\[email protected]\0/ety\[email protected]/fld\[email protected]/cd\[email protected]/blockquote\[email protected]/wordforms\[email protected]/note\0"74          "@/altname\[email protected]/chform\[email protected]/cref\[email protected]/syn\0");75    *ptr2++='>';76    ptr=ptr2;77  } while (res<0);78 79  ncmd_done:80  *_ptr=ptr;81  return res;82}83 84U8 *ACDNextEntry(U8 **_ptr)85{86  U8 *res,*ignore,*ptr=*_ptr,buf[ACD_BLK_SIZE],*out_ptr=buf;87  I64 ch,l;88  while (TRUE) {89    while (TRUE) {90      if (!(ch=*ptr++)) goto nentry_done;91      if (ch!='<') {92        *out_ptr++=ch;93        if (ch=='$')94          *out_ptr++=ch;95      } else96        break;97    }98    ignore="b>\0i>\0ppp>\0/b>\0/i>\0/p>\0"99          "ets>\0col>\0spn>\0/ets>\0/col>\0/spn>\0er>\0as>\0cs>\0cd>\0ex>\0"100          "/er>\0/as>\0/cs>\0/cd>\0/ex>\0"101          "note>\0/note>\0blockquote>\0/blockquote>\0";102    while (*ignore) {103      l=StrLen(ignore);104      if (!StrNCmp(ptr,ignore,l)) {105        ptr+=l;106        break;107      } else108        ignore+=l+1;109    }110    if (!*ignore)111      break;112  }113nentry_done:114  *out_ptr++=0;115  res=StrNew(buf);116  *_ptr=ptr-1;117  return res;118}119 120I64 ACDCompareWords(U8 *e1,U8 *e2)121{122  return StrICmp(e1,e2);123}124 125U8 *ACDSortWords(U8 *start,I64 size,I64 word_cnt)126{127  U8 **ptr_array=MAlloc(sizeof(U8 *)*word_cnt),128        *out_start=MAlloc(size),129        *ptr=start,*ptr2;130  I64 i=0;131  while (*ptr) {132    ptr_array[i++]=ptr;133    ptr+=StrLen(ptr)+3;134  }135  "Sorting...\n"; Sleep(100);136  QSortI64(ptr_array,word_cnt,&ACDCompareWords);137  "Done...\n"; Sleep(100);138 139  ptr=out_start;140  for (i=0;i<word_cnt;i++) {141    ptr2=ptr_array[i];142    while (*ptr2)143      *ptr++=*ptr2++;144    *ptr++=*ptr2++; //zero145    *ptr++=*ptr2++; //blk lo146    *ptr++=*ptr2++; //blk hi147  }148  *ptr++=0;149  return out_start;150}151 152U0 ACDGen(U8 *in_file)153{154  I64 cmd,size,word_cnt=0,largest_entry=0;155  U8 *st,*in_ptr=FileRead(in_file,&size),*in_start=in_ptr,156        *out_ptr=MAlloc(size),*out_start=out_ptr,157        *word_ptr=MAlloc(size),*word_start=word_ptr,158        *last_word="",*def_word_start=out_ptr,159        *sorted_word_start;160  U16 *d;161  if (!in_ptr) return;162  do {163    cmd=ACDNextCmd(&in_ptr);164    if (cmd==ACD_H1) {165next_word:166      if (out_ptr-def_word_start>largest_entry)167        largest_entry=out_ptr-def_word_start;168      def_word_start=out_ptr;169      if (st=ACDNextEntry(&in_ptr)) {170        if (*st) {171          if (StrICmp(st,last_word)) {172            word_cnt++;173 174            *word_ptr++=ACD_WORD_CHAR;175            last_word=word_ptr;176            StrCpy(word_ptr,st);177            word_ptr+=StrLen(st)+1;178 179            d=word_ptr;180            *d=(out_ptr-out_start)/ACD_BLK_SIZE;181            word_ptr+=2;182 183            *out_ptr++=ACD_WORD_CHAR;184            StrCpy(out_ptr,st);185            out_ptr+=StrLen(st)+1;186          }187          Free(st);188 189          do {190            do {191              cmd=ACDNextCmd(&in_ptr);192              if (cmd==ACD_H1)193                goto next_word;194            } while (cmd>=0 && !(cmd==ACD_DEF||cmd==ACD_PRONUNCIATION||195                  cmd==ACD_POS||cmd==ACD_EXTRA));196            if (cmd==ACD_DEF) {197              if(st=ACDNextEntry(&in_ptr)) {198                if (*st) {199                  *out_ptr++=ACD_DEF_CHAR;200                  StrCpy(out_ptr,st);201                  out_ptr+=StrLen(st)+1;202                }203                Free(st);204              }205            } else if (cmd==ACD_PRONUNCIATION) {206              if(st=ACDNextEntry(&in_ptr)) {207                if (*st) {208                  *out_ptr++=ACD_PRONUNCIATION_CHAR;209                  StrCpy(out_ptr,st);210                  out_ptr+=StrLen(st)+1;211                }212                Free(st);213              }214            } else if (cmd==ACD_POS) {215              if(st=ACDNextEntry(&in_ptr)) {216                if (*st) {217                  *out_ptr++=ACD_POS_CHAR;218                  StrCpy(out_ptr,st);219                  out_ptr+=StrLen(st)+1;220                }221                Free(st);222              }223            } else if (cmd==ACD_EXTRA) {224              if(st=ACDNextEntry(&in_ptr)) {225                if (*st) {226                  *out_ptr++=ACD_EXTRA_CHAR;227                  StrCpy(out_ptr,st);228                  out_ptr+=StrLen(st)+1;229                }230                Free(st);231              }232            }233          } while (cmd==ACD_DEF||cmd==ACD_PRONUNCIATION||234                cmd==ACD_POS||cmd==ACD_EXTRA);235        } else236          Free(st);237      }238    }239  } while (cmd>=0);240  *out_ptr++=ACD_END_CHAR;241  *word_ptr++=ACD_END_CHAR;242 243  Free(in_start);244 245  "Blk Size      :%d\n",ACD_BLK_SIZE;246  "Blk Cnt       :%04X\n",(out_ptr-out_start+ACD_BLK_SIZE-1)/ACD_BLK_SIZE;247  "Largest Entry :%d\n",largest_entry;248  "Word Count    :%d\n",word_cnt;249 250  FileWrite(ACD_DEF_FILENAME,out_start,out_ptr-out_start);251  "Def File Size :%d\n",out_ptr-out_start;252 253  sorted_word_start=ACDSortWords(word_start,word_ptr-word_start,word_cnt);254  FileWrite(ACD_WORD_FILENAME,sorted_word_start,word_ptr-word_start);255  "Word File Size:%d\n",word_ptr-word_start;256 257  Free(out_start);258  Free(word_start);259  Free(sorted_word_start);260}261 262Cd(__DIR__);263ACDPreprocess("DICTIONARY.DD","DICTIONARY2.DD");264ACDGen("DICTIONARY2.DD");265