1 <?php
  2 /**
  3  * SilverStripe-variant of the "gettext" tool:
  4  * Parses the string content of all PHP-files and SilverStripe templates
  5  * for ocurrences of the _t() translation method. Also uses the {@link i18nEntityProvider}
  6  * interface to get dynamically defined entities by executing the 
  7  * {@link provideI18nEntities()} method on all implementors of this interface.
  8  * 
  9  * Collects all found entities (and their natural language text for the default locale)
 10  * into language-files for each module in an array notation. Creates or overwrites these files,
 11  * e.g. sapphire/lang/en_US.php.
 12  * 
 13  * The collector needs to be run whenever you make new translatable
 14  * entities available. Please don't alter the arrays in language tables manually.
 15  * 
 16  * Usage through URL: http://localhost/dev/tasks/i18nTextCollectorTast
 17  * Usage through URL (module-specific): http://localhost/dev/tasks/i18nTextCollectorTask/?module=mymodule
 18  * Usage on CLI: sake dev/tasks/i18nTextCollectorTask
 19  * Usage on CLI (module-specific): sake dev/tasks/i18nTextCollectorTask module=mymodule
 20  *
 21  * Requires PHP 5.1+ due to class_implements() limitations
 22  * 
 23  * @author Bernat Foj Capell <bernat@silverstripe.com>
 24  * @author Ingo Schommer <FIRSTNAME@silverstripe.com>
 25  * @package sapphire
 26  * @subpackage i18n
 27  * @uses i18nEntityProvider
 28  * @uses i18n
 29  */
 30 class i18nTextCollector extends Object {
 31     
 32     protected $defaultLocale;
 33     
 34     /**
 35      * @var string $basePath The directory base on which the collector should act.
 36      * Usually the webroot set through {@link Director::baseFolder()}.
 37      * @todo Fully support changing of basePath through {@link SSViewer} and {@link ManifestBuilder}
 38      */
 39     public $basePath;
 40     
 41     /**
 42      * @var string $basePath The directory base on which the collector should create new lang folders and files.
 43      * Usually the webroot set through {@link Director::baseFolder()}.
 44      * Can be overwritten for testing or export purposes.
 45      * @todo Fully support changing of basePath through {@link SSViewer} and {@link ManifestBuilder}
 46      */
 47     public $baseSavePath;
 48     
 49     /**
 50      * @param $locale
 51      */
 52     function __construct($locale = null) {
 53         $this->defaultLocale = ($locale) ? $locale : i18n::default_locale();
 54         $this->basePath = Director::baseFolder();
 55         $this->baseSavePath = Director::baseFolder();
 56         
 57         parent::__construct();
 58     }
 59     
 60     /**
 61      * This is the main method to build the master string tables with the original strings.
 62      * It will search for existent modules that use the i18n feature, parse the _t() calls
 63      * and write the resultant files in the lang folder of each module.
 64      * 
 65      * @uses DataObject->collectI18nStatics()
 66      * 
 67      * @param array $restrictToModules
 68      */ 
 69     public function run($restrictToModules = null) {
 70         //Debug::message("Collecting text...", false);
 71         
 72         $modules = array();
 73         
 74         // A master string tables array (one mst per module)
 75         $entitiesByModule = array();
 76         
 77         //Search for and process existent modules, or use the passed one instead
 78         if($restrictToModules && count($restrictToModules)) {
 79             foreach($restrictToModules as $restrictToModule) {
 80                 $modules[] = basename($restrictToModule);
 81             }
 82         } else {
 83             $modules = scandir($this->basePath);
 84         }
 85 
 86         foreach($modules as $module) {
 87             // Only search for calls in folder with a _config.php file (which means they are modules)  
 88             $isValidModuleFolder = (
 89                 is_dir("$this->basePath/$module") 
 90                 && is_file("$this->basePath/$module/_config.php") 
 91                 && substr($module,0,1) != '.'
 92             );
 93             if(!$isValidModuleFolder) continue;
 94             
 95             // we store the master string tables 
 96             $processedEntities = $this->processModule($module);
 97             if(isset($entitiesByModule[$module])) {
 98                 $entitiesByModule[$module] = array_merge_recursive($entitiesByModule[$module], $processedEntities);
 99             } else {
100                 $entitiesByModule[$module] = $processedEntities;
101             }
102             
103             // extract all entities for "foreign" modules (fourth argument)
104             foreach($entitiesByModule[$module] as $fullName => $spec) {
105                 if(isset($spec[3]) && $spec[3] != $module) {
106                     $othermodule = $spec[3];
107                     if(!isset($entitiesByModule[$othermodule])) $entitiesByModule[$othermodule] = array();
108                     unset($spec[3]);
109                     $entitiesByModule[$othermodule][$fullName] = $spec;
110                     unset($entitiesByModule[$module][$fullName]);
111                 }
112             }
113             
114             // extract all entities for "foreign" modules (fourth argument)
115             foreach($entitiesByModule[$module] as $fullName => $spec) {
116                 if(isset($spec[3]) && $spec[3] != $module) {
117                     $othermodule = $spec[3];
118                     if(!isset($entitiesByModule[$othermodule])) $entitiesByModule[$othermodule] = array();
119                     unset($spec[3]);
120                     $entitiesByModule[$othermodule][$fullName] = $spec;
121                 }
122             }
123         }
124 
125         // Write the generated master string tables
126         $this->writeMasterStringFile($entitiesByModule);
127         
128         //Debug::message("Done!", false);
129     }
130     
131     /**
132      * Build the module's master string table
133      *
134      * @param string $module Module's name
135      */
136     protected function processModule($module) { 
137         $entitiesArr = array();
138 
139         //Debug::message("Processing Module '{$module}'", false);
140 
141         // Search for calls in code files if these exists
142         if(is_dir("$this->basePath/$module/code")) {
143             $fileList = $this->getFilesRecursive("$this->basePath/$module/code");
144         } else if($module == 'sapphire') {
145             // sapphire doesn't have the usual module structure, so we'll scan all subfolders
146             $fileList = $this->getFilesRecursive("$this->basePath/$module");
147         }
148         foreach($fileList as $filePath) {
149             // exclude ss-templates, they're scanned separately
150             if(substr($filePath,-3) == 'php') {
151                 $content = file_get_contents($filePath);
152                 $entitiesArr = array_merge($entitiesArr,(array)$this->collectFromCode($content, $module));
153                 $entitiesArr = array_merge($entitiesArr, (array)$this->collectFromEntityProviders($filePath, $module));
154             }
155         }
156         
157         // Search for calls in template files if these exists
158         if(is_dir("$this->basePath/$module/templates")) {
159             $fileList = $this->getFilesRecursive("$this->basePath/$module/templates");
160             foreach($fileList as $index => $filePath) {
161                 $content = file_get_contents($filePath);
162                 // templates use their filename as a namespace
163                 $namespace = basename($filePath);
164                 $entitiesArr = array_merge($entitiesArr, (array)$this->collectFromTemplate($content, $module, $namespace));
165             }
166         }
167 
168         // sort for easier lookup and comparison with translated files
169         ksort($entitiesArr);
170 
171         return $entitiesArr;
172     }
173     
174     public function collectFromCode($content, $module) {
175         $entitiesArr = array();
176         
177         $regexRule = '_t[[:space:]]*\(' .
178             '[[:space:]]*("[^"]*"|\\\'[^\']*\\\')[[:space:]]*,' . // namespace.entity
179             '[[:space:]]*(("([^"]|\\\")*"|\'([^\']|\\\\\')*\')' .  // value
180             '([[:space:]]*\\.[[:space:]]*("([^"]|\\\")*"|\'([^\']|\\\\\')*\'))*)' . // concatenations
181             '([[:space:]]*,[[:space:]]*[^,)]*)?([[:space:]]*,' . // priority (optional)
182             '[[:space:]]*("([^"]|\\\")*"|\'([^\']|\\\\\')*\'))?[[:space:]]*' . // comment (optional)
183         '\)';
184         
185         while (ereg($regexRule, $content, $regs)) {
186             $entitiesArr = array_merge($entitiesArr, (array)$this->entitySpecFromRegexMatches($regs));
187             
188             // remove parsed content to continue while() loop
189             $content = str_replace($regs[0],"",$content);
190         }
191         
192         ksort($entitiesArr);
193         
194         return $entitiesArr;
195     }
196 
197     public function collectFromTemplate($content, $module, $fileName) {
198         $entitiesArr = array();
199         
200         // Search for included templates
201         preg_match_all('/<' . '% include +([A-Za-z0-9_]+) +%' . '>/', $content, $regs, PREG_SET_ORDER);
202         foreach($regs as $reg) {
203             $includeName = $reg[1];
204             $includeFileName = "{$includeName}.ss";
205             $filePath = SSViewer::getTemplateFileByType($includeName, 'Includes');
206             if(!$filePath) $filePath = SSViewer::getTemplateFileByType($includeName, 'main');
207             if($filePath) {
208                 $includeContent = file_get_contents($filePath);
209                 $entitiesArr = array_merge($entitiesArr,(array)$this->collectFromTemplate($includeContent, $module, $includeFileName));
210             }
211             // @todo Will get massively confused if you include the includer -> infinite loop
212         }
213 
214         // @todo respect template tags (< % _t() % > instead of _t())
215         $regexRule = '_t[[:space:]]*\(' .
216             '[[:space:]]*("[^"]*"|\\\'[^\']*\\\')[[:space:]]*,' . // namespace.entity
217             '[[:space:]]*(("([^"]|\\\")*"|\'([^\']|\\\\\')*\')' .  // value
218             '([[:space:]]*\\.[[:space:]]*("([^"]|\\\")*"|\'([^\']|\\\\\')*\'))*)' . // concatenations
219             '([[:space:]]*,[[:space:]]*[^,)]*)?([[:space:]]*,' . // priority (optional)
220             '[[:space:]]*("([^"]|\\\")*"|\'([^\']|\\\\\')*\'))?[[:space:]]*' . // comment (optional)
221         '\)';
222         while(ereg($regexRule,$content,$regs)) {
223             $entitiesArr = array_merge($entitiesArr,(array)$this->entitySpecFromRegexMatches($regs, $fileName));
224             // remove parsed content to continue while() loop
225             $content = str_replace($regs[0],"",$content);
226         }
227         
228         ksort($entitiesArr);
229         
230         return $entitiesArr;
231     }
232     
233     /**
234      * @uses i18nEntityProvider
235      */
236     function collectFromEntityProviders($filePath) {
237         $entitiesArr = array();
238         
239         $classes = ClassInfo::classes_for_file($filePath);
240         if($classes) foreach($classes as $class) {
241             // Not all classes can be instanciated without mandatory arguments,
242             // so entity collection doesn't work for all SilverStripe classes currently
243             // Requires PHP 5.1+
244             if(class_exists($class) && in_array('i18nEntityProvider', class_implements($class))) {
245                 $reflectionClass = new ReflectionClass($class);
246                 if($reflectionClass->isAbstract()) continue;
247 
248                 $obj = singleton($class);
249                 $entitiesArr = array_merge($entitiesArr,(array)$obj->provideI18nEntities());
250             }
251         }
252         
253         ksort($entitiesArr);
254         
255         return $entitiesArr;
256     }
257     
258     /**
259      * @todo Fix regexes so the deletion of quotes, commas and newlines from wrong matches isn't necessary
260      */
261     protected function entitySpecFromRegexMatches($regs, $_namespace = null) {
262         // remove wrapping quotes
263         $fullName = substr($regs[1],1,-1);
264         
265         // split fullname into entity parts
266         $entityParts = explode('.', $fullName);
267         if(count($entityParts) > 1) {
268             // templates don't have a custom namespace
269             $entity = array_pop($entityParts);
270             // namespace might contain dots, so we explode
271             $namespace = implode('.',$entityParts); 
272         } else {
273             $entity = array_pop($entityParts);
274             $namespace = $_namespace;
275         }
276         
277         // If a dollar sign is used in the entity name,
278         // we can't resolve without running the method,
279         // and skip the processing. This is mostly used for
280         // dynamically translating static properties, e.g. looping
281         // through $db, which are detected by {@link collectFromEntityProviders}.
282         if(strpos('$', $entity) !== FALSE) return false;
283         
284         // remove wrapping quotes
285         $value = ($regs[2]) ? substr($regs[2],1,-1) : null;
286 
287         $value = ereg_replace("([^\\])['\"][[:space:]]*\.[[:space:]]*['\"]",'\\1',$value);
288 
289         // only escape quotes when wrapped in double quotes, to make them safe for insertion
290         // into single-quoted PHP code. If they're wrapped in single quotes, the string should
291         // be properly escaped already
292         if(substr($regs[2],0,1) == '"') {
293             // Double quotes don't need escaping
294             $value = str_replace('\\"','"', $value);
295             // But single quotes do
296             $value = str_replace("'","\\'", $value);
297         }
298 
299         
300         // remove starting comma and any newlines
301         $eol = PHP_EOL;
302         $prio = ($regs[10]) ? trim(preg_replace("/$eol/", '', substr($regs[10],1))) : null;
303         
304         // remove wrapping quotes
305         $comment = ($regs[12]) ? substr($regs[12],1,-1) : null;
306 
307         return array(
308             "{$namespace}.{$entity}" => array(
309                 $value,
310                 $prio,
311                 $comment
312             )
313         );
314     }
315     
316     /**
317      * Input for langArrayCodeForEntitySpec() should be suitable for insertion
318      * into single-quoted strings, so needs to be escaped already.
319      * 
320      * @param string $entity The entity name, e.g. CMSMain.BUTTONSAVE
321      */
322     public function langArrayCodeForEntitySpec($entityFullName, $entitySpec) {
323         $php = '';
324         $eol = PHP_EOL;
325         
326         $entityParts = explode('.', $entityFullName);
327         if(count($entityParts) > 1) {
328             // templates don't have a custom namespace
329             $entity = array_pop($entityParts);
330             // namespace might contain dots, so we implode back
331             $namespace = implode('.',$entityParts); 
332         } else {
333             user_error("i18nTextCollector::langArrayCodeForEntitySpec(): Wrong entity format for $entityFullName with values" . var_export($entitySpec, true), E_USER_WARNING);
334             return false;
335         }
336         
337         $value = $entitySpec[0];
338         $prio = (isset($entitySpec[1])) ? addcslashes($entitySpec[1],'\'') : null;
339         $comment = (isset($entitySpec[2])) ? addcslashes($entitySpec[2],'\'') : null;
340         
341         $php .= '$lang[\'' . $this->defaultLocale . '\'][\'' . $namespace . '\'][\'' . $entity . '\'] = ';
342         if ($prio) {
343             $php .= "array($eol\t'" . $value . "',$eol\t" . $prio;
344             if ($comment) {
345                 $php .= ",$eol\t'" . $comment . '\''; 
346             }
347             $php .= "$eol);";
348         } else {
349             $php .= '\'' . $value . '\';';
350         }
351         $php .= "$eol";
352         
353         return $php;
354     }
355     
356     /**
357      * Write the master string table of every processed module
358      */
359     protected function writeMasterStringFile($entitiesByModule) {
360         // Write each module language file
361         if($entitiesByModule) foreach($entitiesByModule as $module => $entities) {
362             $php = '';
363             $eol = PHP_EOL;
364             
365             // Create folder for lang files
366             $langFolder = $this->baseSavePath . '/' . $module . '/lang';
367             if(!file_exists($langFolder)) {
368                 Filesystem::makeFolder($langFolder, Filesystem::$folder_create_mask);
369                 touch($langFolder . '/_manifest_exclude');
370             }
371 
372             // Open the English file and write the Master String Table
373             $langFile = $langFolder . '/' . $this->defaultLocale . '.php';
374             if($fh = fopen($langFile, "w")) {
375                 if($entities) foreach($entities as $fullName => $spec) {
376                     $php .= $this->langArrayCodeForEntitySpec($fullName, $spec);
377                 }
378                 
379                 // test for valid PHP syntax by eval'ing it
380                 try{
381                     eval($php);
382                 } catch(Exception $e) {
383                     user_error('i18nTextCollector->writeMasterStringFile(): Invalid PHP language file. Error: ' . $e->toString(), E_USER_ERROR);
384                 }
385                 
386                 fwrite($fh, "<"."?php{$eol}{$eol}global \$lang;{$eol}{$eol}" . $php . "{$eol}?".">");
387                 fclose($fh);
388                 
389                 //Debug::message("Created file: $langFolder/" . $this->defaultLocale . ".php", false);
390             } else {
391                 user_error("Cannot write language file! Please check permissions of $langFolder/" . $this->defaultLocale . ".php", E_USER_ERROR);
392             }
393         }
394 
395     }
396     
397     /**
398      * Helper function that searches for potential files to be parsed
399      * 
400      * @param string $folder base directory to scan (will scan recursively)
401      * @param array $fileList Array where potential files will be added to
402      */
403     protected function getFilesRecursive($folder, &$fileList = null) {
404         if(!$fileList) $fileList = array();
405         $items = scandir($folder);
406         $isValidFolder = (
407             !in_array('_manifest_exclude', $items)
408             && !preg_match('/\/tests$/', $folder)
409         );
410 
411         if($items && $isValidFolder) foreach($items as $item) {
412             if(substr($item,0,1) == '.') continue;
413             if(substr($item,-4) == '.php') $fileList[substr($item,0,-4)] = "$folder/$item";
414             else if(substr($item,-3) == '.ss') $fileList[$item] = "$folder/$item";
415             else if(is_dir("$folder/$item")) $this->getFilesRecursive("$folder/$item", $fileList);
416         }
417         return $fileList;
418     }
419     
420     public function getDefaultLocale() {
421         return $this->defaultLocale;
422     }
423     
424     public function setDefaultLocale($locale) {
425         $this->defaultLocale = $locale;
426     }
427 }
428 ?>