Common_localize.cpp (18783B)
1 /* 2 =========================================================================== 3 4 Doom 3 BFG Edition GPL Source Code 5 Copyright (C) 1993-2012 id Software LLC, a ZeniMax Media company. 6 7 This file is part of the Doom 3 BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code"). 8 9 Doom 3 BFG Edition Source Code is free software: you can redistribute it and/or modify 10 it under the terms of the GNU General Public License as published by 11 the Free Software Foundation, either version 3 of the License, or 12 (at your option) any later version. 13 14 Doom 3 BFG Edition Source Code is distributed in the hope that it will be useful, 15 but WITHOUT ANY WARRANTY; without even the implied warranty of 16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 GNU General Public License for more details. 18 19 You should have received a copy of the GNU General Public License 20 along with Doom 3 BFG Edition Source Code. If not, see <http://www.gnu.org/licenses/>. 21 22 In addition, the Doom 3 BFG Edition Source Code is also subject to certain additional terms. You should have received a copy of these additional terms immediately following the terms and conditions of the GNU General Public License which accompanied the Doom 3 BFG Edition Source Code. If not, please request a copy in writing from id Software at the address below. 23 24 If you have questions concerning this license or the applicable additional terms, you may contact in writing id Software LLC, c/o ZeniMax Media Inc., Suite 120, Rockville, Maryland 20850 USA. 25 26 =========================================================================== 27 */ 28 29 #include "../idlib/precompiled.h" 30 #pragma hdrstop 31 32 #include "Common_local.h" 33 34 idCVar com_product_lang_ext( "com_product_lang_ext", "1", CVAR_INTEGER | CVAR_SYSTEM | CVAR_ARCHIVE, "Extension to use when creating language files." ); 35 36 /* 37 ================= 38 LoadMapLocalizeData 39 ================= 40 */ 41 typedef idHashTable<idStrList> ListHash; 42 void LoadMapLocalizeData(ListHash& listHash) { 43 44 idStr fileName = "map_localize.cfg"; 45 const char *buffer = NULL; 46 idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT ); 47 48 if ( fileSystem->ReadFile( fileName, (void**)&buffer ) > 0 ) { 49 src.LoadMemory( buffer, strlen(buffer), fileName ); 50 if ( src.IsLoaded() ) { 51 idStr classname; 52 idToken token; 53 54 55 56 while ( src.ReadToken( &token ) ) { 57 classname = token; 58 src.ExpectTokenString( "{" ); 59 60 idStrList list; 61 while ( src.ReadToken( &token) ) { 62 if ( token == "}" ) { 63 break; 64 } 65 list.Append(token); 66 } 67 68 listHash.Set(classname, list); 69 } 70 } 71 fileSystem->FreeFile( (void*)buffer ); 72 } 73 74 } 75 76 void LoadGuiParmExcludeList(idStrList& list) { 77 78 idStr fileName = "guiparm_exclude.cfg"; 79 const char *buffer = NULL; 80 idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT ); 81 82 if ( fileSystem->ReadFile( fileName, (void**)&buffer ) > 0 ) { 83 src.LoadMemory( buffer, strlen(buffer), fileName ); 84 if ( src.IsLoaded() ) { 85 idStr classname; 86 idToken token; 87 88 89 90 while ( src.ReadToken( &token ) ) { 91 list.Append(token); 92 } 93 } 94 fileSystem->FreeFile( (void*)buffer ); 95 } 96 } 97 98 bool TestMapVal(idStr& str) { 99 //Already Localized? 100 if(str.Find("#str_") != -1) { 101 return false; 102 } 103 104 return true; 105 } 106 107 bool TestGuiParm(const char* parm, const char* value, idStrList& excludeList) { 108 109 idStr testVal = value; 110 111 //Already Localized? 112 if(testVal.Find("#str_") != -1) { 113 return false; 114 } 115 116 //Numeric 117 if(testVal.IsNumeric()) { 118 return false; 119 } 120 121 //Contains :: 122 if(testVal.Find("::") != -1) { 123 return false; 124 } 125 126 //Contains / 127 if(testVal.Find("/") != -1) { 128 return false; 129 } 130 131 if(excludeList.Find(testVal)) { 132 return false; 133 } 134 135 return true; 136 } 137 138 void GetFileList(const char* dir, const char* ext, idStrList& list) { 139 140 //Recurse Subdirectories 141 idStrList dirList; 142 Sys_ListFiles(dir, "/", dirList); 143 for(int i = 0; i < dirList.Num(); i++) { 144 if(dirList[i] == "." || dirList[i] == "..") { 145 continue; 146 } 147 idStr fullName = va("%s/%s", dir, dirList[i].c_str()); 148 GetFileList(fullName, ext, list); 149 } 150 151 idStrList fileList; 152 Sys_ListFiles(dir, ext, fileList); 153 for(int i = 0; i < fileList.Num(); i++) { 154 idStr fullName = va("%s/%s", dir, fileList[i].c_str()); 155 list.Append(fullName); 156 } 157 } 158 159 int LocalizeMap(const char* mapName, idLangDict &langDict, ListHash& listHash, idStrList& excludeList, bool writeFile) { 160 161 common->Printf("Localizing Map '%s'\n", mapName); 162 163 int strCount = 0; 164 165 idMapFile map; 166 if ( map.Parse(mapName, false, false ) ) { 167 int count = map.GetNumEntities(); 168 for ( int j = 0; j < count; j++ ) { 169 idMapEntity *ent = map.GetEntity( j ); 170 if ( ent ) { 171 172 idStr classname = ent->epairs.GetString("classname"); 173 174 //Hack: for info_location 175 bool hasLocation = false; 176 177 idStrList* list; 178 listHash.Get(classname, &list); 179 if(list) { 180 181 for(int k = 0; k < list->Num(); k++) { 182 183 idStr val = ent->epairs.GetString((*list)[k], ""); 184 185 if(val.Length() && classname == "info_location" && (*list)[k] == "location") { 186 hasLocation = true; 187 } 188 189 if(val.Length() && TestMapVal(val)) { 190 191 if(!hasLocation || (*list)[k] == "location") { 192 //Localize it!!! 193 strCount++; 194 ent->epairs.Set( (*list)[k], langDict.AddString( val ) ); 195 } 196 } 197 } 198 } 199 200 listHash.Get("all", &list); 201 if(list) { 202 for(int k = 0; k < list->Num(); k++) { 203 idStr val = ent->epairs.GetString((*list)[k], ""); 204 if(val.Length() && TestMapVal(val)) { 205 //Localize it!!! 206 strCount++; 207 ent->epairs.Set( (*list)[k], langDict.AddString( val ) ); 208 } 209 } 210 } 211 212 //Localize the gui_parms 213 const idKeyValue* kv = ent->epairs.MatchPrefix("gui_parm"); 214 while( kv ) { 215 if(TestGuiParm(kv->GetKey(), kv->GetValue(), excludeList)) { 216 //Localize It! 217 strCount++; 218 ent->epairs.Set( kv->GetKey(), langDict.AddString( kv->GetValue() ) ); 219 } 220 kv = ent->epairs.MatchPrefix( "gui_parm", kv ); 221 } 222 } 223 } 224 if(writeFile && strCount > 0) { 225 //Before we write the map file lets make a backup of the original 226 idStr file = fileSystem->RelativePathToOSPath(mapName); 227 idStr bak = file.Left(file.Length() - 4); 228 bak.Append(".bak_loc"); 229 fileSystem->CopyFile( file, bak ); 230 231 map.Write( mapName, ".map" ); 232 } 233 } 234 235 common->Printf("Count: %d\n", strCount); 236 return strCount; 237 } 238 239 /* 240 ================= 241 LocalizeMaps_f 242 ================= 243 */ 244 CONSOLE_COMMAND( localizeMaps, "localize maps", NULL ) { 245 if ( args.Argc() < 2 ) { 246 common->Printf( "Usage: localizeMaps <count | dictupdate | all> <map>\n" ); 247 return; 248 } 249 250 int strCount = 0; 251 252 bool count = false; 253 bool dictUpdate = false; 254 bool write = false; 255 256 if ( idStr::Icmp( args.Argv(1), "count" ) == 0 ) { 257 count = true; 258 } else if ( idStr::Icmp( args.Argv(1), "dictupdate" ) == 0 ) { 259 count = true; 260 dictUpdate = true; 261 } else if ( idStr::Icmp( args.Argv(1), "all" ) == 0 ) { 262 count = true; 263 dictUpdate = true; 264 write = true; 265 } else { 266 common->Printf( "Invalid Command\n" ); 267 common->Printf( "Usage: localizeMaps <count | dictupdate | all>\n" ); 268 return; 269 270 } 271 272 idLangDict strTable; 273 idStr filename = va("strings/english%.3i.lang", com_product_lang_ext.GetInteger()); 274 275 { 276 // I think this is equivalent... 277 const byte * buffer = NULL; 278 int len = fileSystem->ReadFile( filename, (void**)&buffer ); 279 if ( verify( len > 0 ) ) { 280 strTable.Load( buffer, len, filename ); 281 } 282 fileSystem->FreeFile( (void *)buffer ); 283 284 // ... to this 285 //if ( strTable.Load( filename ) == false) { 286 // //This is a new file so set the base index 287 // strTable.SetBaseID(com_product_lang_ext.GetInteger()*100000); 288 //} 289 } 290 291 common->SetRefreshOnPrint( true ); 292 293 ListHash listHash; 294 LoadMapLocalizeData(listHash); 295 296 idStrList excludeList; 297 LoadGuiParmExcludeList(excludeList); 298 299 if(args.Argc() == 3) { 300 strCount += LocalizeMap(args.Argv(2), strTable, listHash, excludeList, write); 301 } else { 302 idStrList files; 303 GetFileList("z:/d3xp/d3xp/maps/game", "*.map", files); 304 for ( int i = 0; i < files.Num(); i++ ) { 305 idStr file = fileSystem->OSPathToRelativePath(files[i]); 306 strCount += LocalizeMap(file, strTable, listHash, excludeList, write); 307 } 308 } 309 310 if(count) { 311 common->Printf("Localize String Count: %d\n", strCount); 312 } 313 314 common->SetRefreshOnPrint( false ); 315 316 if(dictUpdate) { 317 strTable.Save( filename ); 318 } 319 } 320 321 /* 322 ================= 323 LocalizeGuis_f 324 ================= 325 */ 326 CONSOLE_COMMAND( localizeGuis, "localize guis", NULL ) { 327 328 if ( args.Argc() != 2 ) { 329 common->Printf( "Usage: localizeGuis <all | gui>\n" ); 330 return; 331 } 332 333 idLangDict strTable; 334 335 idStr filename = va("strings/english%.3i.lang", com_product_lang_ext.GetInteger()); 336 337 { 338 // I think this is equivalent... 339 const byte * buffer = NULL; 340 int len = fileSystem->ReadFile( filename, (void**)&buffer ); 341 if ( verify( len > 0 ) ) { 342 strTable.Load( buffer, len, filename ); 343 } 344 fileSystem->FreeFile( (void *)buffer ); 345 346 // ... to this 347 //if(strTable.Load( filename ) == false) { 348 // //This is a new file so set the base index 349 // strTable.SetBaseID(com_product_lang_ext.GetInteger()*100000); 350 //} 351 } 352 353 idFileList *files; 354 if ( idStr::Icmp( args.Argv(1), "all" ) == 0 ) { 355 idStr game = cvarSystem->GetCVarString( "game_expansion" ); 356 if(game.Length()) { 357 files = fileSystem->ListFilesTree( "guis", "*.gui", true, game ); 358 } else { 359 files = fileSystem->ListFilesTree( "guis", "*.gui", true ); 360 } 361 for ( int i = 0; i < files->GetNumFiles(); i++ ) { 362 commonLocal.LocalizeGui( files->GetFile( i ), strTable ); 363 } 364 fileSystem->FreeFileList( files ); 365 366 if(game.Length()) { 367 files = fileSystem->ListFilesTree( "guis", "*.pd", true, game ); 368 } else { 369 files = fileSystem->ListFilesTree( "guis", "*.pd", true, "d3xp" ); 370 } 371 372 for ( int i = 0; i < files->GetNumFiles(); i++ ) { 373 commonLocal.LocalizeGui( files->GetFile( i ), strTable ); 374 } 375 fileSystem->FreeFileList( files ); 376 377 } else { 378 commonLocal.LocalizeGui( args.Argv(1), strTable ); 379 } 380 strTable.Save( filename ); 381 } 382 383 CONSOLE_COMMAND( localizeGuiParmsTest, "Create test files that show gui parms localized and ignored.", NULL ) { 384 385 common->SetRefreshOnPrint( true ); 386 387 idFile *localizeFile = fileSystem->OpenFileWrite( "gui_parm_localize.csv" ); 388 idFile *noLocalizeFile = fileSystem->OpenFileWrite( "gui_parm_nolocalize.csv" ); 389 390 idStrList excludeList; 391 LoadGuiParmExcludeList(excludeList); 392 393 idStrList files; 394 GetFileList("z:/d3xp/d3xp/maps/game", "*.map", files); 395 396 for ( int i = 0; i < files.Num(); i++ ) { 397 398 common->Printf("Testing Map '%s'\n", files[i].c_str()); 399 idMapFile map; 400 401 idStr file = fileSystem->OSPathToRelativePath(files[i]); 402 if ( map.Parse(file, false, false ) ) { 403 int count = map.GetNumEntities(); 404 for ( int j = 0; j < count; j++ ) { 405 idMapEntity *ent = map.GetEntity( j ); 406 if ( ent ) { 407 const idKeyValue* kv = ent->epairs.MatchPrefix("gui_parm"); 408 while( kv ) { 409 if(TestGuiParm(kv->GetKey(), kv->GetValue(), excludeList)) { 410 idStr out = va("%s,%s,%s\r\n", kv->GetValue().c_str(), kv->GetKey().c_str(), file.c_str()); 411 localizeFile->Write( out.c_str(), out.Length() ); 412 } else { 413 idStr out = va("%s,%s,%s\r\n", kv->GetValue().c_str(), kv->GetKey().c_str(), file.c_str()); 414 noLocalizeFile->Write( out.c_str(), out.Length() ); 415 } 416 kv = ent->epairs.MatchPrefix( "gui_parm", kv ); 417 } 418 } 419 } 420 } 421 } 422 423 fileSystem->CloseFile( localizeFile ); 424 fileSystem->CloseFile( noLocalizeFile ); 425 426 common->SetRefreshOnPrint( false ); 427 } 428 429 430 CONSOLE_COMMAND( localizeMapsTest, "Create test files that shows which strings will be localized.", NULL ) { 431 432 ListHash listHash; 433 LoadMapLocalizeData(listHash); 434 435 436 common->SetRefreshOnPrint( true ); 437 438 idFile *localizeFile = fileSystem->OpenFileWrite( "map_localize.csv" ); 439 440 idStrList files; 441 GetFileList("z:/d3xp/d3xp/maps/game", "*.map", files); 442 443 for ( int i = 0; i < files.Num(); i++ ) { 444 445 common->Printf("Testing Map '%s'\n", files[i].c_str()); 446 idMapFile map; 447 448 idStr file = fileSystem->OSPathToRelativePath(files[i]); 449 if ( map.Parse(file, false, false ) ) { 450 int count = map.GetNumEntities(); 451 for ( int j = 0; j < count; j++ ) { 452 idMapEntity *ent = map.GetEntity( j ); 453 if ( ent ) { 454 455 //Temp code to get a list of all entity key value pairs 456 /*idStr classname = ent->epairs.GetString("classname"); 457 if(classname == "worldspawn" || classname == "func_static" || classname == "light" || classname == "speaker" || classname.Left(8) == "trigger_") { 458 continue; 459 } 460 for( int i = 0; i < ent->epairs.GetNumKeyVals(); i++) { 461 const idKeyValue* kv = ent->epairs.GetKeyVal(i); 462 idStr out = va("%s,%s,%s,%s\r\n", classname.c_str(), kv->GetKey().c_str(), kv->GetValue().c_str(), file.c_str()); 463 localizeFile->Write( out.c_str(), out.Length() ); 464 }*/ 465 466 idStr classname = ent->epairs.GetString("classname"); 467 468 //Hack: for info_location 469 bool hasLocation = false; 470 471 idStrList* list; 472 listHash.Get(classname, &list); 473 if(list) { 474 475 for(int k = 0; k < list->Num(); k++) { 476 477 idStr val = ent->epairs.GetString((*list)[k], ""); 478 479 if(classname == "info_location" && (*list)[k] == "location") { 480 hasLocation = true; 481 } 482 483 if(val.Length() && TestMapVal(val)) { 484 485 if(!hasLocation || (*list)[k] == "location") { 486 idStr out = va("%s,%s,%s\r\n", val.c_str(), (*list)[k].c_str(), file.c_str()); 487 localizeFile->Write( out.c_str(), out.Length() ); 488 } 489 } 490 } 491 } 492 493 listHash.Get("all", &list); 494 if(list) { 495 for(int k = 0; k < list->Num(); k++) { 496 idStr val = ent->epairs.GetString((*list)[k], ""); 497 if(val.Length() && TestMapVal(val)) { 498 idStr out = va("%s,%s,%s\r\n", val.c_str(), (*list)[k].c_str(), file.c_str()); 499 localizeFile->Write( out.c_str(), out.Length() ); 500 } 501 } 502 } 503 } 504 } 505 } 506 } 507 508 fileSystem->CloseFile( localizeFile ); 509 510 common->SetRefreshOnPrint( false ); 511 } 512 513 /* 514 =============== 515 idCommonLocal::LocalizeSpecificMapData 516 =============== 517 */ 518 void idCommonLocal::LocalizeSpecificMapData( const char *fileName, idLangDict &langDict, const idLangDict &replaceArgs ) { 519 idStr out, ws, work; 520 521 idMapFile map; 522 if ( map.Parse( fileName, false, false ) ) { 523 int count = map.GetNumEntities(); 524 for ( int i = 0; i < count; i++ ) { 525 idMapEntity *ent = map.GetEntity( i ); 526 if ( ent ) { 527 for ( int j = 0; j < replaceArgs.GetNumKeyVals(); j++ ) { 528 const idLangKeyValue *kv = replaceArgs.GetKeyVal( j ); 529 const char *temp = ent->epairs.GetString( kv->key ); 530 if ( ( temp != NULL ) && *temp ) { 531 idStr val = kv->value; 532 if ( val == temp ) { 533 ent->epairs.Set( kv->key, langDict.AddString( temp ) ); 534 } 535 } 536 } 537 } 538 } 539 map.Write( fileName, ".map" ); 540 } 541 } 542 543 /* 544 =============== 545 idCommonLocal::LocalizeMapData 546 =============== 547 */ 548 void idCommonLocal::LocalizeMapData( const char *fileName, idLangDict &langDict ) { 549 const char *buffer = NULL; 550 idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT ); 551 552 common->SetRefreshOnPrint( true ); 553 554 if ( fileSystem->ReadFile( fileName, (void**)&buffer ) > 0 ) { 555 src.LoadMemory( buffer, strlen(buffer), fileName ); 556 if ( src.IsLoaded() ) { 557 common->Printf( "Processing %s\n", fileName ); 558 idStr mapFileName; 559 idToken token, token2; 560 idLangDict replaceArgs; 561 while ( src.ReadToken( &token ) ) { 562 mapFileName = token; 563 replaceArgs.Clear(); 564 src.ExpectTokenString( "{" ); 565 while ( src.ReadToken( &token) ) { 566 if ( token == "}" ) { 567 break; 568 } 569 if ( src.ReadToken( &token2 ) ) { 570 if ( token2 == "}" ) { 571 break; 572 } 573 replaceArgs.AddKeyVal( token, token2 ); 574 } 575 } 576 common->Printf( " localizing map %s...\n", mapFileName.c_str() ); 577 LocalizeSpecificMapData( mapFileName, langDict, replaceArgs ); 578 } 579 } 580 fileSystem->FreeFile( (void*)buffer ); 581 } 582 583 common->SetRefreshOnPrint( false ); 584 } 585 586 /* 587 =============== 588 idCommonLocal::LocalizeGui 589 =============== 590 */ 591 void idCommonLocal::LocalizeGui( const char *fileName, idLangDict &langDict ) { 592 idStr out, ws, work; 593 const char *buffer = NULL; 594 out.Empty(); 595 int k; 596 char ch; 597 char slash = '\\'; 598 char tab = 't'; 599 char nl = 'n'; 600 idLexer src( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_ALLOWMULTICHARLITERALS | LEXFL_ALLOWBACKSLASHSTRINGCONCAT ); 601 if ( fileSystem->ReadFile( fileName, (void**)&buffer ) > 0 ) { 602 src.LoadMemory( buffer, strlen(buffer), fileName ); 603 if ( src.IsLoaded() ) { 604 idFile *outFile = fileSystem->OpenFileWrite( fileName ); 605 common->Printf( "Processing %s\n", fileName ); 606 607 const bool captureToImage = false; 608 UpdateScreen( captureToImage ); 609 idToken token; 610 while( src.ReadToken( &token ) ) { 611 src.GetLastWhiteSpace( ws ); 612 out += ws; 613 if ( token.type == TT_STRING ) { 614 out += va( "\"%s\"", token.c_str() ); 615 } else { 616 out += token; 617 } 618 if ( out.Length() > 200000 ) { 619 outFile->Write( out.c_str(), out.Length() ); 620 out = ""; 621 } 622 work = token.Right( 6 ); 623 if ( token.Icmp( "text" ) == 0 || work.Icmp( "::text" ) == 0 || token.Icmp( "choices" ) == 0 ) { 624 if ( src.ReadToken( &token ) ) { 625 // see if already exists, if so save that id to this position in this file 626 // otherwise add this to the list and save the id to this position in this file 627 src.GetLastWhiteSpace( ws ); 628 out += ws; 629 token = langDict.AddString( token ); 630 out += "\""; 631 for ( k = 0; k < token.Length(); k++ ) { 632 ch = token[k]; 633 if ( ch == '\t' ) { 634 out += slash; 635 out += tab; 636 } else if ( ch == '\n' || ch == '\r' ) { 637 out += slash; 638 out += nl; 639 } else { 640 out += ch; 641 } 642 } 643 out += "\""; 644 } 645 } else if ( token.Icmp( "comment" ) == 0 ) { 646 if ( src.ReadToken( &token ) ) { 647 // need to write these out by hand to preserve any \n's 648 // see if already exists, if so save that id to this position in this file 649 // otherwise add this to the list and save the id to this position in this file 650 src.GetLastWhiteSpace( ws ); 651 out += ws; 652 out += "\""; 653 for ( k = 0; k < token.Length(); k++ ) { 654 ch = token[k]; 655 if ( ch == '\t' ) { 656 out += slash; 657 out += tab; 658 } else if ( ch == '\n' || ch == '\r' ) { 659 out += slash; 660 out += nl; 661 } else { 662 out += ch; 663 } 664 } 665 out += "\""; 666 } 667 } 668 } 669 outFile->Write( out.c_str(), out.Length() ); 670 fileSystem->CloseFile( outFile ); 671 } 672 fileSystem->FreeFile( (void*)buffer ); 673 } 674 }