4 * Detects the language of a given piece of text.
6 * Attempts to detect the language of a sample of text by correlating ranked
7 * 3-gram frequencies to a table of 3-gram frequencies of known languages.
9 * Implements a version of a technique originally proposed by Cavnar & Trenkle
10 * (1994): "N-Gram-Based Text Categorization"
15 * @package Text_LanguageDetect
16 * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
17 * @copyright 2005-2006 Nicholas Pisarro
18 * @license http://www.debian.org/misc/bsd.license BSD
19 * @version SVN: $Id: LanguageDetect.php 322353 2012-01-16 08:41:43Z cweiske $
20 * @link http://pear.php.net/package/Text_LanguageDetect/
21 * @link http://langdetect.blogspot.com/
24 require_once __DIR__
. '/Text/LanguageDetect/Exception.php';
25 require_once __DIR__
. '/Text/LanguageDetect/Parser.php';
26 require_once __DIR__
. '/Text/LanguageDetect/ISO639.php';
29 * Language detection class
31 * Requires the langauge model database (lang.dat) that should have
32 * accompanied this class definition in order to be instantiated.
37 * require_once 'Text/LanguageDetect.php';
39 * $l = new Text_LanguageDetect;
41 * $stdin = fopen('php://stdin', 'r');
43 * echo "Supported languages:\n";
46 * $langs = $l->getLanguages();
47 * } catch (Text_LanguageDetect_Exception $e) {
48 * die($e->getMessage());
52 * echo join(', ', $langs);
54 * while ($line = fgets($stdin)) {
55 * print_r($l->detect($line, 4));
60 * @package Text_LanguageDetect
61 * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
62 * @copyright 2005 Nicholas Pisarro
63 * @license http://www.debian.org/misc/bsd.license BSD
64 * @version Release: @package_version@
65 * @link http://pear.php.net/package/Text_LanguageDetect/
66 * @todo allow users to generate their own language models
68 * @SuppressWarnings(PHPMD)
70 class Text_LanguageDetect
73 * The filename that stores the trigram data for the detector
75 * If this value starts with a slash (/) or a dot (.) the value of
76 * $this->_data_dir will be ignored
81 var $_db_filename = 'lang.dat';
84 * The filename that stores the unicode block definitions
86 * If this value starts with a slash (/) or a dot (.) the value of
87 * $this->_data_dir will be ignored
92 var $_unicode_db_filename = 'unicode_blocks.dat';
97 * Should be set by PEAR installer
102 var $_data_dir = '@data_dir@';
105 * The trigram data for comparison
107 * Will be loaded on start from $this->_db_filename
112 var $_lang_db = array();
115 * stores the map of the trigram data to unicode characters
123 * The size of the trigram data arrays
128 var $_threshold = 300;
131 * the maximum possible score.
133 * needed for score normalization. Different depending on the
134 * perl compatibility setting
138 * @see setPerlCompatible()
143 * Whether or not to simulate perl's Language::Guess exactly
147 * @see setPerlCompatible()
149 var $_perl_compatible = false;
152 * Whether to use the unicode block detection to speed up processing
157 var $_use_unicode_narrowing = true;
160 * stores the result of the clustering operation
164 * @see clusterLanguages()
169 * Which type of "language names" are accepted and returned:
171 * 0 - language name ("english")
172 * 2 - 2-letter ISO 639-1 code ("en")
173 * 3 - 3-letter ISO 639-2 code ("eng")
180 * Will attempt to load the language database. If it fails, you will get
183 function __construct()
185 $data = $this->_readdb($this->_db_filename
);
186 $this->_checkTrigram($data['trigram']);
187 $this->_lang_db
= $data['trigram'];
189 if (isset($data['trigram-unicodemap'])) {
190 $this->_unicode_map
= $data['trigram-unicodemap'];
193 // Not yet implemented:
194 if (isset($data['trigram-clusters'])) {
195 $this->_clusters
= $data['trigram-clusters'];
200 * Returns the path to the location of the database
202 * @param string $fname File name to load
204 * @return string expected path to the language model database
207 function _get_data_loc($fname)
209 if ($fname{0} == '/' ||
$fname{0} == '.') {
210 // if filename starts with a slash, assume it's an absolute pathname
211 // and skip whatever is in $this->_data_dir
214 } elseif ($this->_data_dir
!= '@' . 'data_dir' . '@') {
215 // if the data dir was set by the PEAR installer, use that
216 return $this->_data_dir
. '/Text_LanguageDetect/' . $fname;
219 // assume this was just unpacked somewhere
220 // try the local working directory if otherwise
221 return __DIR__
. '/data/' . $fname;
226 * Loads the language trigram database from filename
228 * Trigram datbase should be a serialize()'d array
230 * @param string $fname the filename where the data is stored
232 * @return array the language model data
233 * @throws Text_LanguageDetect_Exception
236 function _readdb($fname)
238 // finds the correct data dir
239 $fname = $this->_get_data_loc($fname);
242 if (!file_exists($fname)) {
243 throw new Text_LanguageDetect_Exception(
244 'Language database does not exist: ' . $fname,
245 Text_LanguageDetect_Exception
::DB_NOT_FOUND
247 } elseif (!is_readable($fname)) {
248 throw new Text_LanguageDetect_Exception(
249 'Language database is not readable: ' . $fname,
250 Text_LanguageDetect_Exception
::DB_NOT_READABLE
254 return unserialize(file_get_contents($fname));
259 * Checks if this object is ready to detect languages
261 * @param array $trigram Trigram data from database
266 function _checkTrigram($trigram)
268 if (!is_array($trigram)) {
269 if (ini_get('magic_quotes_runtime')) {
270 throw new Text_LanguageDetect_Exception(
271 'Error loading database. Try turning magic_quotes_runtime off.',
272 Text_LanguageDetect_Exception
::MAGIC_QUOTES
275 throw new Text_LanguageDetect_Exception(
276 'Language database is not an array.',
277 Text_LanguageDetect_Exception
::DB_NOT_ARRAY
279 } elseif (empty($trigram)) {
280 throw new Text_LanguageDetect_Exception(
281 'Language database has no elements.',
282 Text_LanguageDetect_Exception
::DB_EMPTY
290 * Pass this function the name of or an array of names of
291 * languages that you don't want considered
293 * If you're only expecting a limited set of languages, this can greatly
294 * speed up processing
296 * @param mixed $omit_list language name or array of names to omit
297 * @param bool $include_only if true will include (rather than
298 * exclude) only those in the list
300 * @return int number of languages successfully deleted
301 * @throws Text_LanguageDetect_Exception
303 public function omitLanguages($omit_list, $include_only = false)
307 $omit_list = $this->_convertFromNameMode($omit_list);
309 if (!$include_only) {
310 // deleting the given languages
311 if (!is_array($omit_list)) {
312 $omit_list = strtolower($omit_list); // case desensitize
313 if (isset($this->_lang_db
[$omit_list])) {
314 unset($this->_lang_db
[$omit_list]);
318 foreach ($omit_list as $omit_lang) {
319 if (isset($this->_lang_db
[$omit_lang])) {
320 unset($this->_lang_db
[$omit_lang]);
327 // deleting all except the given languages
328 if (!is_array($omit_list)) {
329 $omit_list = array($omit_list);
333 foreach ($omit_list as $key => $omit_lang) {
334 $omit_list[$key] = strtolower($omit_lang);
337 foreach (array_keys($this->_lang_db
) as $lang) {
338 if (!in_array($lang, $omit_list)) {
339 unset($this->_lang_db
[$lang]);
345 // reset the cluster cache if the number of languages changes
346 // this will then have to be recalculated
347 if (isset($this->_clusters
) && $deleted > 0) {
348 $this->_clusters
= null;
356 * Returns the number of languages that this object can detect
359 * @return int the number of languages
360 * @throws Text_LanguageDetect_Exception
362 function getLanguageCount()
364 return count($this->_lang_db
);
368 * Checks if the language with the given name exists in the database
370 * @param mixed $lang Language name or array of language names
372 * @return bool true if language model exists
374 public function languageExists($lang)
376 $lang = $this->_convertFromNameMode($lang);
378 if (is_string($lang)) {
379 return isset($this->_lang_db
[strtolower($lang)]);
381 } elseif (is_array($lang)) {
382 foreach ($lang as $test_lang) {
383 if (!isset($this->_lang_db
[strtolower($test_lang)])) {
390 throw new Text_LanguageDetect_Exception(
391 'Unsupported parameter type passed to languageExists()',
392 Text_LanguageDetect_Exception
::PARAM_TYPE
398 * Returns the list of detectable languages
401 * @return array the names of the languages known to this object<<<<<<<
402 * @throws Text_LanguageDetect_Exception
404 function getLanguages()
406 return $this->_convertToNameMode(
407 array_keys($this->_lang_db
)
412 * Make this object behave like Language::Guess
414 * @param bool $setting false to turn off perl compatibility
418 public function setPerlCompatible($setting = true)
420 if (is_bool($setting)) { // input check
421 $this->_perl_compatible
= $setting;
423 if ($setting == true) {
424 $this->_max_score
= $this->_threshold
;
426 $this->_max_score
= 0;
433 * Sets the way how language names are accepted and returned.
435 * @param integer $name_mode One of the following modes:
436 * 0 - language name ("english")
437 * 2 - 2-letter ISO 639-1 code ("en")
438 * 3 - 3-letter ISO 639-2 code ("eng")
442 function setNameMode($name_mode)
444 $this->_name_mode
= $name_mode;
448 * Whether to use unicode block ranges in detection
450 * Should speed up most detections if turned on (detault is on). In some
451 * circumstances it may be slower, such as for large text samples (> 10K)
452 * in languages that use latin scripts. In other cases it should speed up
453 * detection noticeably.
455 * @param bool $setting false to turn off
459 public function useUnicodeBlocks($setting = true)
461 if (is_bool($setting)) {
462 $this->_use_unicode_narrowing
= $setting;
467 * Converts a piece of text into trigrams
469 * @param string $text text to convert
471 * @return array array of trigram frequencies
473 * @deprecated Superceded by the Text_LanguageDetect_Parser class
475 function _trigram($text)
477 $s = new Text_LanguageDetect_Parser($text);
478 $s->prepareTrigram();
479 $s->prepareUnicode(false);
480 $s->setPadStart(!$this->_perl_compatible
);
482 return $s->getTrigramFreqs();
486 * Converts a set of trigrams from frequencies to ranks
488 * Thresholds (cuts off) the list at $this->_threshold
490 * @param array $arr array of trigram
492 * @return array ranks of trigrams
495 function _arr_rank($arr)
498 // sorts alphabetically first as a standard way of breaking rank ties
499 $this->_bub_sort($arr);
501 // below might also work, but seemed to introduce errors in testing
508 foreach ($arr as $key => $value) {
511 // cut off at a standard threshold
512 if ($i >= $this->_threshold
) {
521 * Sorts an array by value breaking ties alphabetically
523 * @param array &$arr the array to sort
528 function _bub_sort(&$arr)
530 // should do the same as this perl statement:
531 // sort { $trigrams{$b} == $trigrams{$a}
532 // ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
534 // needs to sort by both key and value at once
535 // using the key to break ties for the value
537 // converts array into an array of arrays of each key and value
538 // may be a better way of doing this
541 foreach ($arr as $key => $value) {
542 $combined[] = array($key, $value);
545 usort($combined, array($this, '_sort_func'));
547 $replacement = array();
548 foreach ($combined as $key => $value) {
549 list($new_key, $new_value) = $value;
550 $replacement[$new_key] = $new_value;
557 * Sort function used by bubble sort
559 * Callback function for usort().
561 * @param array $a first param passed by usort()
562 * @param array $b second param passed by usort()
564 * @return int 1 if $a is greater, -1 if not
568 function _sort_func($a, $b)
570 // each is actually a key/value pair, so that it can compare using both
571 list($a_key, $a_value) = $a;
572 list($b_key, $b_value) = $b;
574 if ($a_value == $b_value) {
575 // if the values are the same, break ties using the key
576 return strcmp($a_key, $b_key);
579 // if not, just sort normally
580 if ($a_value > $b_value) {
587 // 0 should not be possible because keys must be unique
591 * Calculates a linear rank-order distance statistic between two sets of
594 * Sums the differences in rank for each trigram. If the trigram does not
595 * appear in both, consider it a difference of $this->_threshold.
597 * This distance measure was proposed by Cavnar & Trenkle (1994). Despite
598 * its simplicity it has been shown to be highly accurate for language
599 * identification tasks.
601 * @param array $arr1 the reference set of trigram ranks
602 * @param array $arr2 the target set of trigram ranks
604 * @return int the sum of the differences between the ranks of
605 * the two trigram sets
608 function _distance($arr1, $arr2)
612 foreach ($arr2 as $key => $value) {
613 if (isset($arr1[$key])) {
614 $distance = abs($value - $arr1[$key]);
616 // $this->_threshold sets the maximum possible distance value
617 // for any one pair of trigrams
618 $distance = $this->_threshold
;
620 $sumdist +
= $distance;
625 // todo: there are other distance statistics to try, e.g. relative
626 // entropy, but they're probably more costly to compute
630 * Normalizes the score returned by _distance()
632 * Different if perl compatible or not
634 * @param int $score the score from _distance()
635 * @param int $base_count the number of trigrams being considered
637 * @return float the normalized score
641 function _normalize_score($score, $base_count = null)
643 if ($base_count === null) {
644 $base_count = $this->_threshold
;
647 if (!$this->_perl_compatible
) {
648 return 1 - ($score / $base_count / $this->_threshold
);
650 return floor($score / $base_count);
656 * Detects the closeness of a sample of text to the known languages
658 * Calculates the statistical difference between the text and
659 * the trigrams for each language, normalizes the score then
660 * returns results for all languages in sorted order
662 * If perl compatible, the score is 300-0, 0 being most similar.
663 * Otherwise, it's 0-1 with 1 being most similar.
665 * The $sample text should be at least a few sentences in length;
666 * should be ascii-7 or utf8 encoded, if another and the mbstring extension
667 * is present it will try to detect and convert. However, experience has
668 * shown that mb_detect_encoding() *does not work very well* with at least
669 * some types of encoding.
671 * @param string $sample a sample of text to compare.
672 * @param int $limit if specified, return an array of the most likely
673 * $limit languages and their scores.
675 * @return mixed sorted array of language scores, blank array if no
676 * useable text was found
678 * @throws Text_LanguageDetect_Exception
680 public function detect($sample, $limit = 0)
683 if (!Text_LanguageDetect_Parser
::validateString($sample)) {
687 // check char encoding
688 // (only if mbstring extension is compiled and PHP > 4.0.6)
689 if (function_exists('mb_detect_encoding')
690 && function_exists('mb_convert_encoding')
692 // mb_detect_encoding isn't very reliable, to say the least
693 // detection should still work with a sufficient sample
694 // of ascii characters
695 $encoding = mb_detect_encoding($sample);
697 // mb_detect_encoding() will return FALSE if detection fails
698 // don't attempt conversion if that's the case
699 if ($encoding != 'ASCII' && $encoding != 'UTF-8'
700 && $encoding !== false
702 // verify the encoding exists in mb_list_encodings
703 if (in_array($encoding, mb_list_encodings())) {
704 $sample = mb_convert_encoding($sample, 'UTF-8', $encoding);
709 $sample_obj = new Text_LanguageDetect_Parser($sample);
710 $sample_obj->prepareTrigram();
711 if ($this->_use_unicode_narrowing
) {
712 $sample_obj->prepareUnicode();
714 $sample_obj->setPadStart(!$this->_perl_compatible
);
715 $sample_obj->analyze();
717 $trigram_freqs =& $sample_obj->getTrigramRanks();
718 $trigram_count = count($trigram_freqs);
720 if ($trigram_count == 0) {
726 // use unicode block detection to narrow down the possibilities
727 if ($this->_use_unicode_narrowing
) {
728 $blocks =& $sample_obj->getUnicodeBlocks();
730 if (is_array($blocks)) {
731 $present_blocks = array_keys($blocks);
733 throw new Text_LanguageDetect_Exception(
734 'Error during block detection',
735 Text_LanguageDetect_Exception
::BLOCK_DETECTION
739 $possible_langs = array();
741 foreach ($present_blocks as $blockname) {
742 if (isset($this->_unicode_map
[$blockname])) {
744 $possible_langs = array_merge(
746 array_keys($this->_unicode_map
[$blockname])
749 // todo: faster way to do this?
753 // could also try an intersect operation rather than a union
754 // in other words, choose languages whose trigrams contain
755 // ALL of the unicode blocks found in this sample
756 // would improve speed but would be completely thrown off by an
757 // unexpected character, like an umlaut appearing in english text
759 $possible_langs = array_intersect(
760 array_keys($this->_lang_db
),
761 array_unique($possible_langs)
764 // needs to intersect it with the keys of _lang_db in case
765 // languages have been omitted
768 // or just try 'em all
769 $possible_langs = array_keys($this->_lang_db
);
773 foreach ($possible_langs as $lang) {
774 $scores[$lang] = $this->_normalize_score(
775 $this->_distance($this->_lang_db
[$lang], $trigram_freqs),
782 if ($this->_perl_compatible
) {
788 // todo: drop languages with a score of $this->_max_score?
790 // limit the number of returned scores
791 if ($limit && is_numeric($limit)) {
792 $limited_scores = array();
795 foreach ($scores as $key => $value) {
796 if ($i++
>= $limit) {
800 $limited_scores[$key] = $value;
803 return $this->_convertToNameMode($limited_scores, true);
805 return $this->_convertToNameMode($scores, true);
810 * Returns only the most similar language to the text sample
812 * Calls $this->detect() and returns only the top result
814 * @param string $sample text to detect the language of
816 * @return string the name of the most likely language
817 * or null if no language is similar
819 * @throws Text_LanguageDetect_Exception
821 public function detectSimple($sample)
823 $scores = $this->detect($sample, 1);
825 // if top language has the maximum possible score,
826 // then the top score will have been picked at random
827 if (!is_array($scores) ||
empty($scores)
828 ||
current($scores) == $this->_max_score
837 * Returns an array containing the most similar language and a confidence
840 * Confidence is a simple measure calculated from the similarity score
841 * minus the similarity score from the next most similar language
842 * divided by the highest possible score. Languages that have closely
843 * related cousins (e.g. Norwegian and Danish) should generally have lower
846 * The similarity score answers the question "How likely is the text the
847 * returned language regardless of the other languages considered?" The
848 * confidence score is one way of answering the question "how likely is the
849 * text the detected language relative to the rest of the language model
852 * To see how similar languages are a priori, see languageSimilarity()
854 * @param string $sample text for which language will be detected
856 * @return array most similar language, score and confidence rating
857 * or null if no language is similar
859 * @throws Text_LanguageDetect_Exception
861 public function detectConfidence($sample)
863 $scores = $this->detect($sample, 2);
865 // if most similar language has the max score, it
866 // will have been picked at random
867 if (!is_array($scores) ||
empty($scores)
868 ||
current($scores) == $this->_max_score
873 $arr['language'] = key($scores);
874 $arr['similarity'] = current($scores);
875 if (next($scores) !== false) { // if false then no next element
876 // the goal is to return a higher value if the distance between
877 // the similarity of the first score and the second score is high
879 if ($this->_perl_compatible
) {
880 $arr['confidence'] = (current($scores) - $arr['similarity'])
884 $arr['confidence'] = $arr['similarity'] - current($scores);
889 $arr['confidence'] = null;
896 * Returns the distribution of unicode blocks in a given utf8 string
898 * For the block name of a single char, use unicodeBlockName()
900 * @param string $str input string. Must be ascii or utf8
901 * @param bool $skip_symbols if true, skip ascii digits, symbols and
902 * non-printing characters. Includes spaces,
903 * newlines and common punctutation characters.
906 * @throws Text_LanguageDetect_Exception
908 public function detectUnicodeBlocks($str, $skip_symbols)
910 $skip_symbols = (bool)$skip_symbols;
913 $sample_obj = new Text_LanguageDetect_Parser($str);
914 $sample_obj->prepareUnicode();
915 $sample_obj->prepareTrigram(false);
916 $sample_obj->setUnicodeSkipSymbols($skip_symbols);
917 $sample_obj->analyze();
918 $blocks = $sample_obj->getUnicodeBlocks();
924 * Returns the block name for a given unicode value
926 * If passed a string, will assume it is being passed a UTF8-formatted
927 * character and will automatically convert. Otherwise it will assume it
928 * is being passed a numeric unicode value.
930 * Make sure input is of the correct type!
932 * @param mixed $unicode unicode value or utf8 char
934 * @return mixed the block name string or false if not found
935 * @throws Text_LanguageDetect_Exception
937 public function unicodeBlockName($unicode)
939 if (is_string($unicode)) {
940 // assume it is being passed a utf8 char, so convert it
941 if (self
::utf8strlen($unicode) > 1) {
942 throw new Text_LanguageDetect_Exception(
943 'Pass a single char only to this method',
944 Text_LanguageDetect_Exception
::PARAM_TYPE
947 $unicode = $this->_utf8char2unicode($unicode);
949 } elseif (!is_int($unicode)) {
950 throw new Text_LanguageDetect_Exception(
951 'Input must be of type string or int.',
952 Text_LanguageDetect_Exception
::PARAM_TYPE
956 $blocks = $this->_read_unicode_block_db();
958 $result = $this->_unicode_block_name($unicode, $blocks);
968 * Searches the unicode block database
970 * Returns the block name for a given unicode value. unicodeBlockName() is
971 * the public interface for this function, which does input checks which
972 * this function omits for speed.
974 * @param int $unicode the unicode value
975 * @param array $blocks the block database
976 * @param int $block_count the number of defined blocks in the database
978 * @return mixed Block name, -1 if it failed
979 * @see unicodeBlockName()
982 function _unicode_block_name($unicode, $blocks, $block_count = -1)
984 // for a reference, see
985 // http://www.unicode.org/Public/UNIDATA/Blocks.txt
987 // assume that ascii characters are the most common
988 // so try it first for efficiency
989 if ($unicode <= $blocks[0][1]) {
993 // the optional $block_count param is for efficiency
994 // so we this function doesn't have to run count() every time
995 if ($block_count != -1) {
996 $high = $block_count - 1;
998 $high = count($blocks) - 1;
1001 $low = 1; // start with 1 because ascii was 0
1003 // your average binary search algorithm
1004 while ($low <= $high) {
1005 $mid = floor(($low +
$high) / 2);
1007 if ($unicode < $blocks[$mid][0]) {
1008 // if it's lower than the lower bound
1011 } elseif ($unicode > $blocks[$mid][1]) {
1012 // if it's higher than the upper bound
1017 return $blocks[$mid];
1021 // failed to find the block
1024 // todo: differentiate when it's out of range or when it falls
1025 // into an unassigned range?
1029 * Brings up the unicode block database
1031 * @return array the database of unicode block definitions
1032 * @throws Text_LanguageDetect_Exception
1035 function _read_unicode_block_db()
1037 // since the unicode definitions are always going to be the same,
1038 // might as well share the memory for the db with all other instances
1042 if (!isset($data)) {
1043 $data = $this->_readdb($this->_unicode_db_filename
);
1050 * Calculate the similarities between the language models
1052 * Use this function to see how similar languages are to each other.
1054 * If passed 2 language names, will return just those languages compared.
1055 * If passed 1 language name, will return that language compared to
1057 * If passed none, will return an array of every language model compared
1058 * to every other one.
1060 * @param string $lang1 the name of the first language to be compared
1061 * @param string $lang2 the name of the second language to be compared
1063 * @return array scores of every language compared
1064 * or the score of just the provided languages
1065 * or null if one of the supplied languages does not exist
1066 * @throws Text_LanguageDetect_Exception
1068 public function languageSimilarity($lang1 = null, $lang2 = null)
1070 $lang1 = $this->_convertFromNameMode($lang1);
1071 $lang2 = $this->_convertFromNameMode($lang2);
1072 if ($lang1 != null) {
1073 $lang1 = strtolower($lang1);
1075 // check if language model exists
1076 if (!isset($this->_lang_db
[$lang1])) {
1080 if ($lang2 != null) {
1081 if (!isset($this->_lang_db
[$lang2])) {
1082 // check if language model exists
1086 $lang2 = strtolower($lang2);
1088 // compare just these two languages
1089 return $this->_normalize_score(
1091 $this->_lang_db
[$lang1],
1092 $this->_lang_db
[$lang2]
1097 // compare just $lang1 to all languages
1098 $return_arr = array();
1099 foreach ($this->_lang_db
as $key => $value) {
1100 if ($key != $lang1) {
1101 // don't compare a language to itself
1102 $return_arr[$key] = $this->_normalize_score(
1103 $this->_distance($this->_lang_db
[$lang1], $value)
1114 // compare all languages to each other
1115 $return_arr = array();
1116 foreach (array_keys($this->_lang_db
) as $lang1) {
1117 foreach (array_keys($this->_lang_db
) as $lang2) {
1118 // skip comparing languages to themselves
1119 if ($lang1 != $lang2) {
1121 if (isset($return_arr[$lang2][$lang1])) {
1122 // don't re-calculate what's already been done
1123 $return_arr[$lang1][$lang2]
1124 = $return_arr[$lang2][$lang1];
1128 $return_arr[$lang1][$lang2]
1129 = $this->_normalize_score(
1131 $this->_lang_db
[$lang1],
1132 $this->_lang_db
[$lang2]
1145 * Cluster known languages according to languageSimilarity()
1147 * WARNING: this method is EXPERIMENTAL. It is not recommended for common
1148 * use, and it may disappear or its functionality may change in future
1149 * releases without notice.
1151 * Uses a nearest neighbor technique to generate the maximum possible
1152 * number of dendograms from the similarity data.
1155 * @return array language cluster data
1156 * @throws Text_LanguageDetect_Exception
1157 * @see languageSimilarity()
1158 * @deprecated this function will eventually be removed and placed into
1159 * the model generation class
1161 function clusterLanguages()
1163 // todo: set the maximum number of clusters
1164 // return cached result, if any
1165 if (isset($this->_clusters
)) {
1166 return $this->_clusters
;
1169 $langs = array_keys($this->_lang_db
);
1171 $arr = $this->languageSimilarity();
1175 foreach ($langs as $lang) {
1176 if (!isset($this->_lang_db
[$lang])) {
1177 throw new Text_LanguageDetect_Exception(
1179 Text_LanguageDetect_Exception
::UNKNOWN_LANGUAGE
1184 // http://www.psychstat.missouristate.edu/multibook/mlt04m.html
1185 foreach ($langs as $old_key => $lang1) {
1186 $langs[$lang1] = $lang1;
1187 unset($langs[$old_key]);
1190 $result_data = $really_map = array();
1193 while (count($langs) > 2 && $i++
< 200) {
1194 $highest_score = -1;
1197 foreach ($langs as $lang1) {
1198 foreach ($langs as $lang2) {
1199 if ($lang1 != $lang2
1200 && $arr[$lang1][$lang2] > $highest_score
1202 $highest_score = $arr[$lang1][$lang2];
1203 $highest_key1 = $lang1;
1204 $highest_key2 = $lang2;
1209 if (!$highest_key1) {
1210 // should not ever happen
1211 throw new Text_LanguageDetect_Exception(
1212 "no highest key? (step: $i)",
1213 Text_LanguageDetect_Exception
::NO_HIGHEST_KEY
1217 if ($highest_score == 0) {
1218 // languages are perfectly dissimilar
1222 // $highest_key1 and $highest_key2 are most similar
1223 $sum1 = array_sum($arr[$highest_key1]);
1224 $sum2 = array_sum($arr[$highest_key2]);
1226 // use the score for the one that is most similar to the rest of
1227 // the field as the score for the group
1228 // todo: could try averaging or "centroid" method instead
1229 // seems like that might make more sense
1230 // actually nearest neighbor may be better for binary searching
1233 // for "Complete Linkage"/"furthest neighbor"
1235 // for "Single Linkage"/"nearest neighbor" method
1236 // should should be >
1237 // results seem to be pretty much the same with either method
1239 // figure out which to delete and which to replace
1240 if ($sum1 > $sum2) {
1241 $replaceme = $highest_key1;
1242 $deleteme = $highest_key2;
1244 $replaceme = $highest_key2;
1245 $deleteme = $highest_key1;
1248 $newkey = $replaceme . ':' . $deleteme;
1250 // $replaceme is most similar to remaining languages
1251 // replace $replaceme with '$newkey', deleting $deleteme
1253 // keep a record of which fork is really which language
1254 $really_lang = $replaceme;
1255 while (isset($really_map[$really_lang])) {
1256 $really_lang = $really_map[$really_lang];
1258 $really_map[$newkey] = $really_lang;
1261 // replace the best fitting key, delete the other
1262 foreach ($arr as $key1 => $arr2) {
1263 foreach ($arr2 as $key2 => $value2) {
1264 if ($key2 == $replaceme) {
1265 $arr[$key1][$newkey] = $arr[$key1][$key2];
1266 unset($arr[$key1][$key2]);
1267 // replacing $arr[$key1][$key2] with $arr[$key1][$newkey]
1270 if ($key1 == $replaceme) {
1271 $arr[$newkey][$key2] = $arr[$key1][$key2];
1272 unset($arr[$key1][$key2]);
1273 // replacing $arr[$key1][$key2] with $arr[$newkey][$key2]
1276 if ($key1 == $deleteme ||
$key2 == $deleteme) {
1277 // deleting $arr[$key1][$key2]
1278 unset($arr[$key1][$key2]);
1284 unset($langs[$highest_key1]);
1285 unset($langs[$highest_key2]);
1286 $langs[$newkey] = $newkey;
1289 // some of these may be overkill
1290 $result_data[$newkey] = array(
1291 'newkey' => $newkey,
1293 'diff' => abs($sum1 - $sum2),
1294 'score' => $highest_score,
1295 'bestfit' => $replaceme,
1296 'otherfit' => $deleteme,
1297 'really' => $really_lang,
1301 $return_val = array(
1302 'open_forks' => $langs,
1303 // the top level of clusters
1304 // clusters that are mutually exclusive
1305 // or specified by a specific maximum
1307 'fork_data' => $result_data,
1308 // data for each split
1310 'name_map' => $really_map,
1311 // which cluster is really which language
1312 // using the nearest neighbor technique, the cluster
1313 // inherits all of the properties of its most-similar member
1318 // saves the result in the object
1319 $this->_clusters
= $return_val;
1326 * Perform an intelligent detection based on clusterLanguages()
1328 * WARNING: this method is EXPERIMENTAL. It is not recommended for common
1329 * use, and it may disappear or its functionality may change in future
1330 * releases without notice.
1332 * This compares the sample text to top the top level of clusters. If the
1333 * sample is similar to the cluster it will drop down and compare it to the
1334 * languages in the cluster, and so on until it hits a leaf node.
1336 * this should find the language in considerably fewer compares
1337 * (the equivalent of a binary search), however clusterLanguages() is costly
1338 * and the loss of accuracy from this technique is significant.
1340 * This method may need to be 'fuzzier' in order to become more accurate.
1342 * This function could be more useful if the universe of possible languages
1343 * was very large, however in such cases some method of Bayesian inference
1344 * might be more helpful.
1346 * @param string $str input string
1348 * @return array language scores (only those compared)
1349 * @throws Text_LanguageDetect_Exception
1350 * @see clusterLanguages()
1352 public function clusteredSearch($str)
1355 if (!Text_LanguageDetect_Parser
::validateString($str)) {
1359 // clusterLanguages() will return a cached result if possible
1360 // so it's safe to call it every time
1361 $result = $this->clusterLanguages();
1363 $dendogram_start = $result['open_forks'];
1364 $dendogram_data = $result['fork_data'];
1365 $dendogram_alias = $result['name_map'];
1367 $sample_obj = new Text_LanguageDetect_Parser($str);
1368 $sample_obj->prepareTrigram();
1369 $sample_obj->setPadStart(!$this->_perl_compatible
);
1370 $sample_obj->analyze();
1371 $sample_result = $sample_obj->getTrigramRanks();
1372 $sample_count = count($sample_result);
1375 if ($sample_count == 0) {
1379 $i = 0; // counts the number of steps
1381 foreach ($dendogram_start as $lang) {
1382 if (isset($dendogram_alias[$lang])) {
1383 $lang_key = $dendogram_alias[$lang];
1388 $scores[$lang] = $this->_normalize_score(
1389 $this->_distance($this->_lang_db
[$lang_key], $sample_result),
1396 if ($this->_perl_compatible
) {
1402 $top_score = current($scores);
1403 $top_key = key($scores);
1405 // of starting forks, $top_key is the most similar to the sample
1407 $cur_key = $top_key;
1408 while (isset($dendogram_data[$cur_key])) {
1409 $lang1 = $dendogram_data[$cur_key]['bestfit'];
1410 $lang2 = $dendogram_data[$cur_key]['otherfit'];
1411 foreach (array($lang1, $lang2) as $lang) {
1412 if (isset($dendogram_alias[$lang])) {
1413 $lang_key = $dendogram_alias[$lang];
1418 $scores[$lang] = $this->_normalize_score(
1419 $this->_distance($this->_lang_db
[$lang_key], $sample_result),
1423 //todo: does not need to do same comparison again
1428 if ($scores[$lang1] > $scores[$lang2]) {
1430 $loser_key = $lang2;
1433 $loser_key = $lang1;
1436 $diff = $scores[$cur_key] - $scores[$loser_key];
1438 // $cur_key ({$dendogram_alias[$cur_key]}) wins
1439 // over $loser_key ({$dendogram_alias[$loser_key]})
1440 // with a difference of $diff
1443 // found result in $i compares
1445 // rather than sorting the result, preserve it so that you can see
1446 // which paths the algorithm decided to take along the tree
1448 // but sometimes the last item is only the second highest
1449 if (($this->_perl_compatible
&& (end($scores) > prev($scores)))
1450 ||
(!$this->_perl_compatible
&& (end($scores) < prev($scores)))
1452 $real_last_score = current($scores);
1453 $real_last_key = key($scores);
1455 // swaps the 2nd-to-last item for the last item
1456 unset($scores[$real_last_key]);
1457 $scores[$real_last_key] = $real_last_score;
1461 if (!$this->_perl_compatible
) {
1462 $scores = array_reverse($scores, true);
1463 // second param requires php > 4.0.3
1472 * Returns the numbers of characters (not bytes) in a utf8 string
1474 * @param string $str string to get the length of
1476 * @return int number of chars
1478 public static function utf8strlen($str)
1480 // utf8_decode() will convert unknown chars to '?', which is actually
1481 // ideal for counting.
1483 return strlen(utf8_decode($str));
1485 // idea stolen from dokuwiki
1489 * Returns the unicode value of a utf8 char
1491 * @param string $char a utf8 (possibly multi-byte) char
1493 * @return int unicode value
1495 * @link http://en.wikipedia.org/wiki/UTF-8
1497 function _utf8char2unicode($char)
1499 // strlen() here will actually get the binary length of a single char
1500 switch (strlen($char)) {
1502 // normal ASCII-7 byte
1503 // 0xxxxxxx --> 0xxxxxxx
1504 return ord($char{0});
1508 // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx
1509 $z = (ord($char{0}) & 0x000001F) << 6;
1510 $x = (ord($char{1}) & 0x0000003F);
1515 // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx
1516 $z = (ord($char{0}) & 0x0000000F) << 12;
1517 $x1 = (ord($char{1}) & 0x0000003F) << 6;
1518 $x2 = (ord($char{2}) & 0x0000003F);
1519 return ($z |
$x1 |
$x2);
1523 // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx -->
1524 // 000zzzzz xxxxxxxx xxxxxxxx
1525 $z1 = (ord($char{0}) & 0x00000007) << 18;
1526 $z2 = (ord($char{1}) & 0x0000003F) << 12;
1527 $x1 = (ord($char{2}) & 0x0000003F) << 6;
1528 $x2 = (ord($char{3}) & 0x0000003F);
1529 return ($z1 |
$z2 |
$x1 |
$x2);
1534 * utf8-safe fast character iterator
1536 * Will get the next character starting from $counter, which will then be
1537 * incremented. If a multi-byte char the bytes will be concatenated and
1538 * $counter will be incremeted by the number of bytes in the char.
1540 * @param string $str the string being iterated over
1541 * @param int &$counter the iterator, will increment by reference
1542 * @param bool $special_convert whether to do special conversions
1544 * @return char the next (possibly multi-byte) char from $counter
1547 static function _next_char($str, &$counter, $special_convert = false)
1549 $char = $str{$counter++
};
1552 // for a description of the utf8 system see
1553 // http://www.phpclasses.org/browse/file/5131.html
1555 // normal ascii one byte char
1557 // special conversions needed for this package
1558 // (that only apply to regular ascii characters)
1559 // lower case, and convert all non-alphanumeric characters
1560 // other than "'" to space
1561 if ($special_convert && $char != ' ' && $char != "'") {
1562 if ($ord >= 65 && $ord <= 90) { // A-Z
1563 $char = chr($ord +
32); // lower case
1564 } elseif ($ord < 97 ||
$ord > 122) { // NOT a-z
1565 $char = ' '; // convert to space
1571 } elseif ($ord >> 5 == 6) { // two-byte char
1573 $nextchar = $str{$counter++
}; // get next byte
1575 // lower-casing of non-ascii characters is still incomplete
1577 if ($special_convert) {
1578 // lower case latin accented characters
1580 $nextord = ord($nextchar);
1581 $nextord_adj = $nextord +
64;
1582 // for a reference, see
1583 // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
1585 // À - Þ but not ×
1586 if ($nextord_adj >= 192
1587 && $nextord_adj <= 222
1588 && $nextord_adj != 215
1590 $nextchar = chr($nextord +
32);
1593 } elseif ($ord == 208) {
1594 // lower case cyrillic alphabet
1595 $nextord = ord($nextchar);
1597 if ($nextord >= 144 && $nextord <= 159) {
1599 $nextchar = chr($nextord +
32);
1601 } elseif ($nextord >= 160 && $nextord <= 175) {
1604 $char = chr(209); // == $ord++
1605 $nextchar = chr($nextord - 32);
1611 return $char . $nextchar;
1612 } elseif ($ord >> 4 == 14) { // three-byte char
1614 // tag on next 2 bytes
1615 return $char . $str{$counter++
} . $str{$counter++
};
1617 } elseif ($ord >> 3 == 30) { // four-byte char
1619 // tag on next 3 bytes
1620 return $char . $str{$counter++
} . $str{$counter++
} . $str{$counter++
};
1628 * Converts an $language input parameter from the configured mode
1629 * to the language name that is used internally.
1631 * Works for strings and arrays.
1633 * @param string|array $lang A language description ("english"/"en"/"eng")
1634 * @param boolean $convertKey If $lang is an array, setting $key
1635 * converts the keys to the language name.
1637 * @return string|array Language name
1639 function _convertFromNameMode($lang, $convertKey = false)
1641 if ($this->_name_mode
== 0) {
1645 if ($this->_name_mode
== 2) {
1646 $method = 'code2ToName';
1648 $method = 'code3ToName';
1651 if (is_string($lang)) {
1652 return (string)Text_LanguageDetect_ISO639
::$method($lang);
1656 foreach ($lang as $key => $val) {
1658 $newkey = (string)Text_LanguageDetect_ISO639
::$method($key);
1659 $newlang[$newkey] = $val;
1661 $newlang[$key] = (string)Text_LanguageDetect_ISO639
::$method($val);
1668 * Converts an $language output parameter from the language name that is
1669 * used internally to the configured mode.
1671 * Works for strings and arrays.
1673 * @param string|array $lang A language description ("english"/"en"/"eng")
1674 * @param boolean $convertKey If $lang is an array, setting $key
1675 * converts the keys to the language name.
1677 * @return string|array Language name
1679 function _convertToNameMode($lang, $convertKey = false)
1681 if ($this->_name_mode
== 0) {
1685 if ($this->_name_mode
== 2) {
1686 $method = 'nameToCode2';
1688 $method = 'nameToCode3';
1691 if (is_string($lang)) {
1692 return Text_LanguageDetect_ISO639
::$method($lang);
1696 foreach ($lang as $key => $val) {
1698 $newkey = Text_LanguageDetect_ISO639
::$method($key);
1699 $newlang[$newkey] = $val;
1701 $newlang[$key] = Text_LanguageDetect_ISO639
::$method($val);
1708 /* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */