private $lang;
function about() {
- return array(1.0,
+ return array(1.1,
"Detect article language",
"fox");
}
$host->add_hook($host::HOOK_ARTICLE_FILTER, $this);
- require_once __DIR__ . "/languagedetect/LanguageDetect.php";
+ require_once __DIR__ . "/languagedetect/Text/LanguageDetect.php";
$this->lang = new Text_LanguageDetect();
$this->lang->setNameMode(2);
+++ /dev/null
-<?php
-
-/**
- * Detects the language of a given piece of text.
- *
- * Attempts to detect the language of a sample of text by correlating ranked
- * 3-gram frequencies to a table of 3-gram frequencies of known languages.
- *
- * Implements a version of a technique originally proposed by Cavnar & Trenkle
- * (1994): "N-Gram-Based Text Categorization"
- *
- * PHP version 5
- *
- * @category Text
- * @package Text_LanguageDetect
- * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
- * @copyright 2005-2006 Nicholas Pisarro
- * @license http://www.debian.org/misc/bsd.license BSD
- * @version SVN: $Id: LanguageDetect.php 322353 2012-01-16 08:41:43Z cweiske $
- * @link http://pear.php.net/package/Text_LanguageDetect/
- * @link http://langdetect.blogspot.com/
- */
-
-require_once __DIR__ . '/Text/LanguageDetect/Exception.php';
-require_once __DIR__ . '/Text/LanguageDetect/Parser.php';
-require_once __DIR__ . '/Text/LanguageDetect/ISO639.php';
-
-/**
- * Language detection class
- *
- * Requires the langauge model database (lang.dat) that should have
- * accompanied this class definition in order to be instantiated.
- *
- * Example usage:
- *
- * <code>
- * require_once 'Text/LanguageDetect.php';
- *
- * $l = new Text_LanguageDetect;
- *
- * $stdin = fopen('php://stdin', 'r');
- *
- * echo "Supported languages:\n";
- *
- * try {
- * $langs = $l->getLanguages();
- * } catch (Text_LanguageDetect_Exception $e) {
- * die($e->getMessage());
- * }
- *
- * sort($langs);
- * echo join(', ', $langs);
- *
- * while ($line = fgets($stdin)) {
- * print_r($l->detect($line, 4));
- * }
- * </code>
- *
- * @category Text
- * @package Text_LanguageDetect
- * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
- * @copyright 2005 Nicholas Pisarro
- * @license http://www.debian.org/misc/bsd.license BSD
- * @version Release: @package_version@
- * @link http://pear.php.net/package/Text_LanguageDetect/
- * @todo allow users to generate their own language models
- *
- * @SuppressWarnings(PHPMD)
- */
-class Text_LanguageDetect
-{
- /**
- * The filename that stores the trigram data for the detector
- *
- * If this value starts with a slash (/) or a dot (.) the value of
- * $this->_data_dir will be ignored
- *
- * @var string
- * @access private
- */
- var $_db_filename = 'lang.dat';
-
- /**
- * The filename that stores the unicode block definitions
- *
- * If this value starts with a slash (/) or a dot (.) the value of
- * $this->_data_dir will be ignored
- *
- * @var string
- * @access private
- */
- var $_unicode_db_filename = 'unicode_blocks.dat';
-
- /**
- * The data directory
- *
- * Should be set by PEAR installer
- *
- * @var string
- * @access private
- */
- var $_data_dir = '@data_dir@';
-
- /**
- * The trigram data for comparison
- *
- * Will be loaded on start from $this->_db_filename
- *
- * @var array
- * @access private
- */
- var $_lang_db = array();
-
- /**
- * stores the map of the trigram data to unicode characters
- *
- * @access private
- * @var array
- */
- var $_unicode_map;
-
- /**
- * The size of the trigram data arrays
- *
- * @var int
- * @access private
- */
- var $_threshold = 300;
-
- /**
- * the maximum possible score.
- *
- * needed for score normalization. Different depending on the
- * perl compatibility setting
- *
- * @access private
- * @var int
- * @see setPerlCompatible()
- */
- var $_max_score = 0;
-
- /**
- * Whether or not to simulate perl's Language::Guess exactly
- *
- * @access private
- * @var bool
- * @see setPerlCompatible()
- */
- var $_perl_compatible = false;
-
- /**
- * Whether to use the unicode block detection to speed up processing
- *
- * @access private
- * @var bool
- */
- var $_use_unicode_narrowing = true;
-
- /**
- * stores the result of the clustering operation
- *
- * @access private
- * @var array
- * @see clusterLanguages()
- */
- var $_clusters;
-
- /**
- * Which type of "language names" are accepted and returned:
- *
- * 0 - language name ("english")
- * 2 - 2-letter ISO 639-1 code ("en")
- * 3 - 3-letter ISO 639-2 code ("eng")
- */
- var $_name_mode = 0;
-
- /**
- * Constructor
- *
- * Will attempt to load the language database. If it fails, you will get
- * an exception.
- */
- function __construct()
- {
- $data = $this->_readdb($this->_db_filename);
- $this->_checkTrigram($data['trigram']);
- $this->_lang_db = $data['trigram'];
-
- if (isset($data['trigram-unicodemap'])) {
- $this->_unicode_map = $data['trigram-unicodemap'];
- }
-
- // Not yet implemented:
- if (isset($data['trigram-clusters'])) {
- $this->_clusters = $data['trigram-clusters'];
- }
- }
-
- /**
- * Returns the path to the location of the database
- *
- * @param string $fname File name to load
- *
- * @return string expected path to the language model database
- * @access private
- */
- function _get_data_loc($fname)
- {
- if ($fname{0} == '/' || $fname{0} == '.') {
- // if filename starts with a slash, assume it's an absolute pathname
- // and skip whatever is in $this->_data_dir
- return $fname;
-
- } elseif ($this->_data_dir != '@' . 'data_dir' . '@') {
- // if the data dir was set by the PEAR installer, use that
- return $this->_data_dir . '/Text_LanguageDetect/' . $fname;
-
- } else {
- // assume this was just unpacked somewhere
- // try the local working directory if otherwise
- return __DIR__ . '/data/' . $fname;
- }
- }
-
- /**
- * Loads the language trigram database from filename
- *
- * Trigram datbase should be a serialize()'d array
- *
- * @param string $fname the filename where the data is stored
- *
- * @return array the language model data
- * @throws Text_LanguageDetect_Exception
- * @access private
- */
- function _readdb($fname)
- {
- // finds the correct data dir
- $fname = $this->_get_data_loc($fname);
-
- // input check
- if (!file_exists($fname)) {
- throw new Text_LanguageDetect_Exception(
- 'Language database does not exist: ' . $fname,
- Text_LanguageDetect_Exception::DB_NOT_FOUND
- );
- } elseif (!is_readable($fname)) {
- throw new Text_LanguageDetect_Exception(
- 'Language database is not readable: ' . $fname,
- Text_LanguageDetect_Exception::DB_NOT_READABLE
- );
- }
-
- return unserialize(file_get_contents($fname));
- }
-
-
- /**
- * Checks if this object is ready to detect languages
- *
- * @param array $trigram Trigram data from database
- *
- * @return void
- * @access private
- */
- function _checkTrigram($trigram)
- {
- if (!is_array($trigram)) {
- if (ini_get('magic_quotes_runtime')) {
- throw new Text_LanguageDetect_Exception(
- 'Error loading database. Try turning magic_quotes_runtime off.',
- Text_LanguageDetect_Exception::MAGIC_QUOTES
- );
- }
- throw new Text_LanguageDetect_Exception(
- 'Language database is not an array.',
- Text_LanguageDetect_Exception::DB_NOT_ARRAY
- );
- } elseif (empty($trigram)) {
- throw new Text_LanguageDetect_Exception(
- 'Language database has no elements.',
- Text_LanguageDetect_Exception::DB_EMPTY
- );
- }
- }
-
- /**
- * Omits languages
- *
- * Pass this function the name of or an array of names of
- * languages that you don't want considered
- *
- * If you're only expecting a limited set of languages, this can greatly
- * speed up processing
- *
- * @param mixed $omit_list language name or array of names to omit
- * @param bool $include_only if true will include (rather than
- * exclude) only those in the list
- *
- * @return int number of languages successfully deleted
- * @throws Text_LanguageDetect_Exception
- */
- public function omitLanguages($omit_list, $include_only = false)
- {
- $deleted = 0;
-
- $omit_list = $this->_convertFromNameMode($omit_list);
-
- if (!$include_only) {
- // deleting the given languages
- if (!is_array($omit_list)) {
- $omit_list = strtolower($omit_list); // case desensitize
- if (isset($this->_lang_db[$omit_list])) {
- unset($this->_lang_db[$omit_list]);
- $deleted++;
- }
- } else {
- foreach ($omit_list as $omit_lang) {
- if (isset($this->_lang_db[$omit_lang])) {
- unset($this->_lang_db[$omit_lang]);
- $deleted++;
- }
- }
- }
-
- } else {
- // deleting all except the given languages
- if (!is_array($omit_list)) {
- $omit_list = array($omit_list);
- }
-
- // case desensitize
- foreach ($omit_list as $key => $omit_lang) {
- $omit_list[$key] = strtolower($omit_lang);
- }
-
- foreach (array_keys($this->_lang_db) as $lang) {
- if (!in_array($lang, $omit_list)) {
- unset($this->_lang_db[$lang]);
- $deleted++;
- }
- }
- }
-
- // reset the cluster cache if the number of languages changes
- // this will then have to be recalculated
- if (isset($this->_clusters) && $deleted > 0) {
- $this->_clusters = null;
- }
-
- return $deleted;
- }
-
-
- /**
- * Returns the number of languages that this object can detect
- *
- * @access public
- * @return int the number of languages
- * @throws Text_LanguageDetect_Exception
- */
- function getLanguageCount()
- {
- return count($this->_lang_db);
- }
-
- /**
- * Checks if the language with the given name exists in the database
- *
- * @param mixed $lang Language name or array of language names
- *
- * @return bool true if language model exists
- */
- public function languageExists($lang)
- {
- $lang = $this->_convertFromNameMode($lang);
-
- if (is_string($lang)) {
- return isset($this->_lang_db[strtolower($lang)]);
-
- } elseif (is_array($lang)) {
- foreach ($lang as $test_lang) {
- if (!isset($this->_lang_db[strtolower($test_lang)])) {
- return false;
- }
- }
- return true;
-
- } else {
- throw new Text_LanguageDetect_Exception(
- 'Unsupported parameter type passed to languageExists()',
- Text_LanguageDetect_Exception::PARAM_TYPE
- );
- }
- }
-
- /**
- * Returns the list of detectable languages
- *
- * @access public
- * @return array the names of the languages known to this object<<<<<<<
- * @throws Text_LanguageDetect_Exception
- */
- function getLanguages()
- {
- return $this->_convertToNameMode(
- array_keys($this->_lang_db)
- );
- }
-
- /**
- * Make this object behave like Language::Guess
- *
- * @param bool $setting false to turn off perl compatibility
- *
- * @return void
- */
- public function setPerlCompatible($setting = true)
- {
- if (is_bool($setting)) { // input check
- $this->_perl_compatible = $setting;
-
- if ($setting == true) {
- $this->_max_score = $this->_threshold;
- } else {
- $this->_max_score = 0;
- }
- }
-
- }
-
- /**
- * Sets the way how language names are accepted and returned.
- *
- * @param integer $name_mode One of the following modes:
- * 0 - language name ("english")
- * 2 - 2-letter ISO 639-1 code ("en")
- * 3 - 3-letter ISO 639-2 code ("eng")
- *
- * @return void
- */
- function setNameMode($name_mode)
- {
- $this->_name_mode = $name_mode;
- }
-
- /**
- * Whether to use unicode block ranges in detection
- *
- * Should speed up most detections if turned on (detault is on). In some
- * circumstances it may be slower, such as for large text samples (> 10K)
- * in languages that use latin scripts. In other cases it should speed up
- * detection noticeably.
- *
- * @param bool $setting false to turn off
- *
- * @return void
- */
- public function useUnicodeBlocks($setting = true)
- {
- if (is_bool($setting)) {
- $this->_use_unicode_narrowing = $setting;
- }
- }
-
- /**
- * Converts a piece of text into trigrams
- *
- * @param string $text text to convert
- *
- * @return array array of trigram frequencies
- * @access private
- * @deprecated Superceded by the Text_LanguageDetect_Parser class
- */
- function _trigram($text)
- {
- $s = new Text_LanguageDetect_Parser($text);
- $s->prepareTrigram();
- $s->prepareUnicode(false);
- $s->setPadStart(!$this->_perl_compatible);
- $s->analyze();
- return $s->getTrigramFreqs();
- }
-
- /**
- * Converts a set of trigrams from frequencies to ranks
- *
- * Thresholds (cuts off) the list at $this->_threshold
- *
- * @param array $arr array of trigram
- *
- * @return array ranks of trigrams
- * @access protected
- */
- function _arr_rank($arr)
- {
-
- // sorts alphabetically first as a standard way of breaking rank ties
- $this->_bub_sort($arr);
-
- // below might also work, but seemed to introduce errors in testing
- //ksort($arr);
- //asort($arr);
-
- $rank = array();
-
- $i = 0;
- foreach ($arr as $key => $value) {
- $rank[$key] = $i++;
-
- // cut off at a standard threshold
- if ($i >= $this->_threshold) {
- break;
- }
- }
-
- return $rank;
- }
-
- /**
- * Sorts an array by value breaking ties alphabetically
- *
- * @param array &$arr the array to sort
- *
- * @return void
- * @access private
- */
- function _bub_sort(&$arr)
- {
- // should do the same as this perl statement:
- // sort { $trigrams{$b} == $trigrams{$a}
- // ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
-
- // needs to sort by both key and value at once
- // using the key to break ties for the value
-
- // converts array into an array of arrays of each key and value
- // may be a better way of doing this
- $combined = array();
-
- foreach ($arr as $key => $value) {
- $combined[] = array($key, $value);
- }
-
- usort($combined, array($this, '_sort_func'));
-
- $replacement = array();
- foreach ($combined as $key => $value) {
- list($new_key, $new_value) = $value;
- $replacement[$new_key] = $new_value;
- }
-
- $arr = $replacement;
- }
-
- /**
- * Sort function used by bubble sort
- *
- * Callback function for usort().
- *
- * @param array $a first param passed by usort()
- * @param array $b second param passed by usort()
- *
- * @return int 1 if $a is greater, -1 if not
- * @see _bub_sort()
- * @access private
- */
- function _sort_func($a, $b)
- {
- // each is actually a key/value pair, so that it can compare using both
- list($a_key, $a_value) = $a;
- list($b_key, $b_value) = $b;
-
- if ($a_value == $b_value) {
- // if the values are the same, break ties using the key
- return strcmp($a_key, $b_key);
-
- } else {
- // if not, just sort normally
- if ($a_value > $b_value) {
- return -1;
- } else {
- return 1;
- }
- }
-
- // 0 should not be possible because keys must be unique
- }
-
- /**
- * Calculates a linear rank-order distance statistic between two sets of
- * ranked trigrams
- *
- * Sums the differences in rank for each trigram. If the trigram does not
- * appear in both, consider it a difference of $this->_threshold.
- *
- * This distance measure was proposed by Cavnar & Trenkle (1994). Despite
- * its simplicity it has been shown to be highly accurate for language
- * identification tasks.
- *
- * @param array $arr1 the reference set of trigram ranks
- * @param array $arr2 the target set of trigram ranks
- *
- * @return int the sum of the differences between the ranks of
- * the two trigram sets
- * @access private
- */
- function _distance($arr1, $arr2)
- {
- $sumdist = 0;
-
- foreach ($arr2 as $key => $value) {
- if (isset($arr1[$key])) {
- $distance = abs($value - $arr1[$key]);
- } else {
- // $this->_threshold sets the maximum possible distance value
- // for any one pair of trigrams
- $distance = $this->_threshold;
- }
- $sumdist += $distance;
- }
-
- return $sumdist;
-
- // todo: there are other distance statistics to try, e.g. relative
- // entropy, but they're probably more costly to compute
- }
-
- /**
- * Normalizes the score returned by _distance()
- *
- * Different if perl compatible or not
- *
- * @param int $score the score from _distance()
- * @param int $base_count the number of trigrams being considered
- *
- * @return float the normalized score
- * @see _distance()
- * @access private
- */
- function _normalize_score($score, $base_count = null)
- {
- if ($base_count === null) {
- $base_count = $this->_threshold;
- }
-
- if (!$this->_perl_compatible) {
- return 1 - ($score / $base_count / $this->_threshold);
- } else {
- return floor($score / $base_count);
- }
- }
-
-
- /**
- * Detects the closeness of a sample of text to the known languages
- *
- * Calculates the statistical difference between the text and
- * the trigrams for each language, normalizes the score then
- * returns results for all languages in sorted order
- *
- * If perl compatible, the score is 300-0, 0 being most similar.
- * Otherwise, it's 0-1 with 1 being most similar.
- *
- * The $sample text should be at least a few sentences in length;
- * should be ascii-7 or utf8 encoded, if another and the mbstring extension
- * is present it will try to detect and convert. However, experience has
- * shown that mb_detect_encoding() *does not work very well* with at least
- * some types of encoding.
- *
- * @param string $sample a sample of text to compare.
- * @param int $limit if specified, return an array of the most likely
- * $limit languages and their scores.
- *
- * @return mixed sorted array of language scores, blank array if no
- * useable text was found
- * @see _distance()
- * @throws Text_LanguageDetect_Exception
- */
- public function detect($sample, $limit = 0)
- {
- // input check
- if (!Text_LanguageDetect_Parser::validateString($sample)) {
- return array();
- }
-
- // check char encoding
- // (only if mbstring extension is compiled and PHP > 4.0.6)
- if (function_exists('mb_detect_encoding')
- && function_exists('mb_convert_encoding')
- ) {
- // mb_detect_encoding isn't very reliable, to say the least
- // detection should still work with a sufficient sample
- // of ascii characters
- $encoding = mb_detect_encoding($sample);
-
- // mb_detect_encoding() will return FALSE if detection fails
- // don't attempt conversion if that's the case
- if ($encoding != 'ASCII' && $encoding != 'UTF-8'
- && $encoding !== false
- ) {
- // verify the encoding exists in mb_list_encodings
- if (in_array($encoding, mb_list_encodings())) {
- $sample = mb_convert_encoding($sample, 'UTF-8', $encoding);
- }
- }
- }
-
- $sample_obj = new Text_LanguageDetect_Parser($sample);
- $sample_obj->prepareTrigram();
- if ($this->_use_unicode_narrowing) {
- $sample_obj->prepareUnicode();
- }
- $sample_obj->setPadStart(!$this->_perl_compatible);
- $sample_obj->analyze();
-
- $trigram_freqs =& $sample_obj->getTrigramRanks();
- $trigram_count = count($trigram_freqs);
-
- if ($trigram_count == 0) {
- return array();
- }
-
- $scores = array();
-
- // use unicode block detection to narrow down the possibilities
- if ($this->_use_unicode_narrowing) {
- $blocks =& $sample_obj->getUnicodeBlocks();
-
- if (is_array($blocks)) {
- $present_blocks = array_keys($blocks);
- } else {
- throw new Text_LanguageDetect_Exception(
- 'Error during block detection',
- Text_LanguageDetect_Exception::BLOCK_DETECTION
- );
- }
-
- $possible_langs = array();
-
- foreach ($present_blocks as $blockname) {
- if (isset($this->_unicode_map[$blockname])) {
-
- $possible_langs = array_merge(
- $possible_langs,
- array_keys($this->_unicode_map[$blockname])
- );
-
- // todo: faster way to do this?
- }
- }
-
- // could also try an intersect operation rather than a union
- // in other words, choose languages whose trigrams contain
- // ALL of the unicode blocks found in this sample
- // would improve speed but would be completely thrown off by an
- // unexpected character, like an umlaut appearing in english text
-
- $possible_langs = array_intersect(
- array_keys($this->_lang_db),
- array_unique($possible_langs)
- );
-
- // needs to intersect it with the keys of _lang_db in case
- // languages have been omitted
-
- } else {
- // or just try 'em all
- $possible_langs = array_keys($this->_lang_db);
- }
-
-
- foreach ($possible_langs as $lang) {
- $scores[$lang] = $this->_normalize_score(
- $this->_distance($this->_lang_db[$lang], $trigram_freqs),
- $trigram_count
- );
- }
-
- unset($sample_obj);
-
- if ($this->_perl_compatible) {
- asort($scores);
- } else {
- arsort($scores);
- }
-
- // todo: drop languages with a score of $this->_max_score?
-
- // limit the number of returned scores
- if ($limit && is_numeric($limit)) {
- $limited_scores = array();
-
- $i = 0;
- foreach ($scores as $key => $value) {
- if ($i++ >= $limit) {
- break;
- }
-
- $limited_scores[$key] = $value;
- }
-
- return $this->_convertToNameMode($limited_scores, true);
- } else {
- return $this->_convertToNameMode($scores, true);
- }
- }
-
- /**
- * Returns only the most similar language to the text sample
- *
- * Calls $this->detect() and returns only the top result
- *
- * @param string $sample text to detect the language of
- *
- * @return string the name of the most likely language
- * or null if no language is similar
- * @see detect()
- * @throws Text_LanguageDetect_Exception
- */
- public function detectSimple($sample)
- {
- $scores = $this->detect($sample, 1);
-
- // if top language has the maximum possible score,
- // then the top score will have been picked at random
- if (!is_array($scores) || empty($scores)
- || current($scores) == $this->_max_score
- ) {
- return null;
- } else {
- return key($scores);
- }
- }
-
- /**
- * Returns an array containing the most similar language and a confidence
- * rating
- *
- * Confidence is a simple measure calculated from the similarity score
- * minus the similarity score from the next most similar language
- * divided by the highest possible score. Languages that have closely
- * related cousins (e.g. Norwegian and Danish) should generally have lower
- * confidence scores.
- *
- * The similarity score answers the question "How likely is the text the
- * returned language regardless of the other languages considered?" The
- * confidence score is one way of answering the question "how likely is the
- * text the detected language relative to the rest of the language model
- * set?"
- *
- * To see how similar languages are a priori, see languageSimilarity()
- *
- * @param string $sample text for which language will be detected
- *
- * @return array most similar language, score and confidence rating
- * or null if no language is similar
- * @see detect()
- * @throws Text_LanguageDetect_Exception
- */
- public function detectConfidence($sample)
- {
- $scores = $this->detect($sample, 2);
-
- // if most similar language has the max score, it
- // will have been picked at random
- if (!is_array($scores) || empty($scores)
- || current($scores) == $this->_max_score
- ) {
- return null;
- }
-
- $arr['language'] = key($scores);
- $arr['similarity'] = current($scores);
- if (next($scores) !== false) { // if false then no next element
- // the goal is to return a higher value if the distance between
- // the similarity of the first score and the second score is high
-
- if ($this->_perl_compatible) {
- $arr['confidence'] = (current($scores) - $arr['similarity'])
- / $this->_max_score;
-
- } else {
- $arr['confidence'] = $arr['similarity'] - current($scores);
-
- }
-
- } else {
- $arr['confidence'] = null;
- }
-
- return $arr;
- }
-
- /**
- * Returns the distribution of unicode blocks in a given utf8 string
- *
- * For the block name of a single char, use unicodeBlockName()
- *
- * @param string $str input string. Must be ascii or utf8
- * @param bool $skip_symbols if true, skip ascii digits, symbols and
- * non-printing characters. Includes spaces,
- * newlines and common punctutation characters.
- *
- * @return array
- * @throws Text_LanguageDetect_Exception
- */
- public function detectUnicodeBlocks($str, $skip_symbols)
- {
- $skip_symbols = (bool)$skip_symbols;
- $str = (string)$str;
-
- $sample_obj = new Text_LanguageDetect_Parser($str);
- $sample_obj->prepareUnicode();
- $sample_obj->prepareTrigram(false);
- $sample_obj->setUnicodeSkipSymbols($skip_symbols);
- $sample_obj->analyze();
- $blocks = $sample_obj->getUnicodeBlocks();
- unset($sample_obj);
- return $blocks;
- }
-
- /**
- * Returns the block name for a given unicode value
- *
- * If passed a string, will assume it is being passed a UTF8-formatted
- * character and will automatically convert. Otherwise it will assume it
- * is being passed a numeric unicode value.
- *
- * Make sure input is of the correct type!
- *
- * @param mixed $unicode unicode value or utf8 char
- *
- * @return mixed the block name string or false if not found
- * @throws Text_LanguageDetect_Exception
- */
- public function unicodeBlockName($unicode)
- {
- if (is_string($unicode)) {
- // assume it is being passed a utf8 char, so convert it
- if (self::utf8strlen($unicode) > 1) {
- throw new Text_LanguageDetect_Exception(
- 'Pass a single char only to this method',
- Text_LanguageDetect_Exception::PARAM_TYPE
- );
- }
- $unicode = $this->_utf8char2unicode($unicode);
-
- } elseif (!is_int($unicode)) {
- throw new Text_LanguageDetect_Exception(
- 'Input must be of type string or int.',
- Text_LanguageDetect_Exception::PARAM_TYPE
- );
- }
-
- $blocks = $this->_read_unicode_block_db();
-
- $result = $this->_unicode_block_name($unicode, $blocks);
-
- if ($result == -1) {
- return false;
- } else {
- return $result[2];
- }
- }
-
- /**
- * Searches the unicode block database
- *
- * Returns the block name for a given unicode value. unicodeBlockName() is
- * the public interface for this function, which does input checks which
- * this function omits for speed.
- *
- * @param int $unicode the unicode value
- * @param array $blocks the block database
- * @param int $block_count the number of defined blocks in the database
- *
- * @return mixed Block name, -1 if it failed
- * @see unicodeBlockName()
- * @access protected
- */
- function _unicode_block_name($unicode, $blocks, $block_count = -1)
- {
- // for a reference, see
- // http://www.unicode.org/Public/UNIDATA/Blocks.txt
-
- // assume that ascii characters are the most common
- // so try it first for efficiency
- if ($unicode <= $blocks[0][1]) {
- return $blocks[0];
- }
-
- // the optional $block_count param is for efficiency
- // so we this function doesn't have to run count() every time
- if ($block_count != -1) {
- $high = $block_count - 1;
- } else {
- $high = count($blocks) - 1;
- }
-
- $low = 1; // start with 1 because ascii was 0
-
- // your average binary search algorithm
- while ($low <= $high) {
- $mid = floor(($low + $high) / 2);
-
- if ($unicode < $blocks[$mid][0]) {
- // if it's lower than the lower bound
- $high = $mid - 1;
-
- } elseif ($unicode > $blocks[$mid][1]) {
- // if it's higher than the upper bound
- $low = $mid + 1;
-
- } else {
- // found it
- return $blocks[$mid];
- }
- }
-
- // failed to find the block
- return -1;
-
- // todo: differentiate when it's out of range or when it falls
- // into an unassigned range?
- }
-
- /**
- * Brings up the unicode block database
- *
- * @return array the database of unicode block definitions
- * @throws Text_LanguageDetect_Exception
- * @access protected
- */
- function _read_unicode_block_db()
- {
- // since the unicode definitions are always going to be the same,
- // might as well share the memory for the db with all other instances
- // of this class
- static $data;
-
- if (!isset($data)) {
- $data = $this->_readdb($this->_unicode_db_filename);
- }
-
- return $data;
- }
-
- /**
- * Calculate the similarities between the language models
- *
- * Use this function to see how similar languages are to each other.
- *
- * If passed 2 language names, will return just those languages compared.
- * If passed 1 language name, will return that language compared to
- * all others.
- * If passed none, will return an array of every language model compared
- * to every other one.
- *
- * @param string $lang1 the name of the first language to be compared
- * @param string $lang2 the name of the second language to be compared
- *
- * @return array scores of every language compared
- * or the score of just the provided languages
- * or null if one of the supplied languages does not exist
- * @throws Text_LanguageDetect_Exception
- */
- public function languageSimilarity($lang1 = null, $lang2 = null)
- {
- $lang1 = $this->_convertFromNameMode($lang1);
- $lang2 = $this->_convertFromNameMode($lang2);
- if ($lang1 != null) {
- $lang1 = strtolower($lang1);
-
- // check if language model exists
- if (!isset($this->_lang_db[$lang1])) {
- return null;
- }
-
- if ($lang2 != null) {
- if (!isset($this->_lang_db[$lang2])) {
- // check if language model exists
- return null;
- }
-
- $lang2 = strtolower($lang2);
-
- // compare just these two languages
- return $this->_normalize_score(
- $this->_distance(
- $this->_lang_db[$lang1],
- $this->_lang_db[$lang2]
- )
- );
-
- } else {
- // compare just $lang1 to all languages
- $return_arr = array();
- foreach ($this->_lang_db as $key => $value) {
- if ($key != $lang1) {
- // don't compare a language to itself
- $return_arr[$key] = $this->_normalize_score(
- $this->_distance($this->_lang_db[$lang1], $value)
- );
- }
- }
- asort($return_arr);
-
- return $return_arr;
- }
-
-
- } else {
- // compare all languages to each other
- $return_arr = array();
- foreach (array_keys($this->_lang_db) as $lang1) {
- foreach (array_keys($this->_lang_db) as $lang2) {
- // skip comparing languages to themselves
- if ($lang1 != $lang2) {
-
- if (isset($return_arr[$lang2][$lang1])) {
- // don't re-calculate what's already been done
- $return_arr[$lang1][$lang2]
- = $return_arr[$lang2][$lang1];
-
- } else {
- // calculate
- $return_arr[$lang1][$lang2]
- = $this->_normalize_score(
- $this->_distance(
- $this->_lang_db[$lang1],
- $this->_lang_db[$lang2]
- )
- );
-
- }
- }
- }
- }
- return $return_arr;
- }
- }
-
- /**
- * Cluster known languages according to languageSimilarity()
- *
- * WARNING: this method is EXPERIMENTAL. It is not recommended for common
- * use, and it may disappear or its functionality may change in future
- * releases without notice.
- *
- * Uses a nearest neighbor technique to generate the maximum possible
- * number of dendograms from the similarity data.
- *
- * @access public
- * @return array language cluster data
- * @throws Text_LanguageDetect_Exception
- * @see languageSimilarity()
- * @deprecated this function will eventually be removed and placed into
- * the model generation class
- */
- function clusterLanguages()
- {
- // todo: set the maximum number of clusters
- // return cached result, if any
- if (isset($this->_clusters)) {
- return $this->_clusters;
- }
-
- $langs = array_keys($this->_lang_db);
-
- $arr = $this->languageSimilarity();
-
- sort($langs);
-
- foreach ($langs as $lang) {
- if (!isset($this->_lang_db[$lang])) {
- throw new Text_LanguageDetect_Exception(
- "missing $lang!",
- Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE
- );
- }
- }
-
- // http://www.psychstat.missouristate.edu/multibook/mlt04m.html
- foreach ($langs as $old_key => $lang1) {
- $langs[$lang1] = $lang1;
- unset($langs[$old_key]);
- }
-
- $result_data = $really_map = array();
-
- $i = 0;
- while (count($langs) > 2 && $i++ < 200) {
- $highest_score = -1;
- $highest_key1 = '';
- $highest_key2 = '';
- foreach ($langs as $lang1) {
- foreach ($langs as $lang2) {
- if ($lang1 != $lang2
- && $arr[$lang1][$lang2] > $highest_score
- ) {
- $highest_score = $arr[$lang1][$lang2];
- $highest_key1 = $lang1;
- $highest_key2 = $lang2;
- }
- }
- }
-
- if (!$highest_key1) {
- // should not ever happen
- throw new Text_LanguageDetect_Exception(
- "no highest key? (step: $i)",
- Text_LanguageDetect_Exception::NO_HIGHEST_KEY
- );
- }
-
- if ($highest_score == 0) {
- // languages are perfectly dissimilar
- break;
- }
-
- // $highest_key1 and $highest_key2 are most similar
- $sum1 = array_sum($arr[$highest_key1]);
- $sum2 = array_sum($arr[$highest_key2]);
-
- // use the score for the one that is most similar to the rest of
- // the field as the score for the group
- // todo: could try averaging or "centroid" method instead
- // seems like that might make more sense
- // actually nearest neighbor may be better for binary searching
-
-
- // for "Complete Linkage"/"furthest neighbor"
- // sign should be <
- // for "Single Linkage"/"nearest neighbor" method
- // should should be >
- // results seem to be pretty much the same with either method
-
- // figure out which to delete and which to replace
- if ($sum1 > $sum2) {
- $replaceme = $highest_key1;
- $deleteme = $highest_key2;
- } else {
- $replaceme = $highest_key2;
- $deleteme = $highest_key1;
- }
-
- $newkey = $replaceme . ':' . $deleteme;
-
- // $replaceme is most similar to remaining languages
- // replace $replaceme with '$newkey', deleting $deleteme
-
- // keep a record of which fork is really which language
- $really_lang = $replaceme;
- while (isset($really_map[$really_lang])) {
- $really_lang = $really_map[$really_lang];
- }
- $really_map[$newkey] = $really_lang;
-
-
- // replace the best fitting key, delete the other
- foreach ($arr as $key1 => $arr2) {
- foreach ($arr2 as $key2 => $value2) {
- if ($key2 == $replaceme) {
- $arr[$key1][$newkey] = $arr[$key1][$key2];
- unset($arr[$key1][$key2]);
- // replacing $arr[$key1][$key2] with $arr[$key1][$newkey]
- }
-
- if ($key1 == $replaceme) {
- $arr[$newkey][$key2] = $arr[$key1][$key2];
- unset($arr[$key1][$key2]);
- // replacing $arr[$key1][$key2] with $arr[$newkey][$key2]
- }
-
- if ($key1 == $deleteme || $key2 == $deleteme) {
- // deleting $arr[$key1][$key2]
- unset($arr[$key1][$key2]);
- }
- }
- }
-
-
- unset($langs[$highest_key1]);
- unset($langs[$highest_key2]);
- $langs[$newkey] = $newkey;
-
-
- // some of these may be overkill
- $result_data[$newkey] = array(
- 'newkey' => $newkey,
- 'count' => $i,
- 'diff' => abs($sum1 - $sum2),
- 'score' => $highest_score,
- 'bestfit' => $replaceme,
- 'otherfit' => $deleteme,
- 'really' => $really_lang,
- );
- }
-
- $return_val = array(
- 'open_forks' => $langs,
- // the top level of clusters
- // clusters that are mutually exclusive
- // or specified by a specific maximum
-
- 'fork_data' => $result_data,
- // data for each split
-
- 'name_map' => $really_map,
- // which cluster is really which language
- // using the nearest neighbor technique, the cluster
- // inherits all of the properties of its most-similar member
- // this keeps track
- );
-
-
- // saves the result in the object
- $this->_clusters = $return_val;
-
- return $return_val;
- }
-
-
- /**
- * Perform an intelligent detection based on clusterLanguages()
- *
- * WARNING: this method is EXPERIMENTAL. It is not recommended for common
- * use, and it may disappear or its functionality may change in future
- * releases without notice.
- *
- * This compares the sample text to top the top level of clusters. If the
- * sample is similar to the cluster it will drop down and compare it to the
- * languages in the cluster, and so on until it hits a leaf node.
- *
- * this should find the language in considerably fewer compares
- * (the equivalent of a binary search), however clusterLanguages() is costly
- * and the loss of accuracy from this technique is significant.
- *
- * This method may need to be 'fuzzier' in order to become more accurate.
- *
- * This function could be more useful if the universe of possible languages
- * was very large, however in such cases some method of Bayesian inference
- * might be more helpful.
- *
- * @param string $str input string
- *
- * @return array language scores (only those compared)
- * @throws Text_LanguageDetect_Exception
- * @see clusterLanguages()
- */
- public function clusteredSearch($str)
- {
- // input check
- if (!Text_LanguageDetect_Parser::validateString($str)) {
- return array();
- }
-
- // clusterLanguages() will return a cached result if possible
- // so it's safe to call it every time
- $result = $this->clusterLanguages();
-
- $dendogram_start = $result['open_forks'];
- $dendogram_data = $result['fork_data'];
- $dendogram_alias = $result['name_map'];
-
- $sample_obj = new Text_LanguageDetect_Parser($str);
- $sample_obj->prepareTrigram();
- $sample_obj->setPadStart(!$this->_perl_compatible);
- $sample_obj->analyze();
- $sample_result = $sample_obj->getTrigramRanks();
- $sample_count = count($sample_result);
-
- // input check
- if ($sample_count == 0) {
- return array();
- }
-
- $i = 0; // counts the number of steps
-
- foreach ($dendogram_start as $lang) {
- if (isset($dendogram_alias[$lang])) {
- $lang_key = $dendogram_alias[$lang];
- } else {
- $lang_key = $lang;
- }
-
- $scores[$lang] = $this->_normalize_score(
- $this->_distance($this->_lang_db[$lang_key], $sample_result),
- $sample_count
- );
-
- $i++;
- }
-
- if ($this->_perl_compatible) {
- asort($scores);
- } else {
- arsort($scores);
- }
-
- $top_score = current($scores);
- $top_key = key($scores);
-
- // of starting forks, $top_key is the most similar to the sample
-
- $cur_key = $top_key;
- while (isset($dendogram_data[$cur_key])) {
- $lang1 = $dendogram_data[$cur_key]['bestfit'];
- $lang2 = $dendogram_data[$cur_key]['otherfit'];
- foreach (array($lang1, $lang2) as $lang) {
- if (isset($dendogram_alias[$lang])) {
- $lang_key = $dendogram_alias[$lang];
- } else {
- $lang_key = $lang;
- }
-
- $scores[$lang] = $this->_normalize_score(
- $this->_distance($this->_lang_db[$lang_key], $sample_result),
- $sample_count
- );
-
- //todo: does not need to do same comparison again
- }
-
- $i++;
-
- if ($scores[$lang1] > $scores[$lang2]) {
- $cur_key = $lang1;
- $loser_key = $lang2;
- } else {
- $cur_key = $lang2;
- $loser_key = $lang1;
- }
-
- $diff = $scores[$cur_key] - $scores[$loser_key];
-
- // $cur_key ({$dendogram_alias[$cur_key]}) wins
- // over $loser_key ({$dendogram_alias[$loser_key]})
- // with a difference of $diff
- }
-
- // found result in $i compares
-
- // rather than sorting the result, preserve it so that you can see
- // which paths the algorithm decided to take along the tree
-
- // but sometimes the last item is only the second highest
- if (($this->_perl_compatible && (end($scores) > prev($scores)))
- || (!$this->_perl_compatible && (end($scores) < prev($scores)))
- ) {
- $real_last_score = current($scores);
- $real_last_key = key($scores);
-
- // swaps the 2nd-to-last item for the last item
- unset($scores[$real_last_key]);
- $scores[$real_last_key] = $real_last_score;
- }
-
-
- if (!$this->_perl_compatible) {
- $scores = array_reverse($scores, true);
- // second param requires php > 4.0.3
- }
-
- return $scores;
- }
-
- /**
- * ut8-safe strlen()
- *
- * Returns the numbers of characters (not bytes) in a utf8 string
- *
- * @param string $str string to get the length of
- *
- * @return int number of chars
- */
- public static function utf8strlen($str)
- {
- // utf8_decode() will convert unknown chars to '?', which is actually
- // ideal for counting.
-
- return strlen(utf8_decode($str));
-
- // idea stolen from dokuwiki
- }
-
- /**
- * Returns the unicode value of a utf8 char
- *
- * @param string $char a utf8 (possibly multi-byte) char
- *
- * @return int unicode value
- * @access protected
- * @link http://en.wikipedia.org/wiki/UTF-8
- */
- function _utf8char2unicode($char)
- {
- // strlen() here will actually get the binary length of a single char
- switch (strlen($char)) {
- case 1:
- // normal ASCII-7 byte
- // 0xxxxxxx --> 0xxxxxxx
- return ord($char{0});
-
- case 2:
- // 2 byte unicode
- // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx
- $z = (ord($char{0}) & 0x000001F) << 6;
- $x = (ord($char{1}) & 0x0000003F);
- return ($z | $x);
-
- case 3:
- // 3 byte unicode
- // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx
- $z = (ord($char{0}) & 0x0000000F) << 12;
- $x1 = (ord($char{1}) & 0x0000003F) << 6;
- $x2 = (ord($char{2}) & 0x0000003F);
- return ($z | $x1 | $x2);
-
- case 4:
- // 4 byte unicode
- // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx -->
- // 000zzzzz xxxxxxxx xxxxxxxx
- $z1 = (ord($char{0}) & 0x00000007) << 18;
- $z2 = (ord($char{1}) & 0x0000003F) << 12;
- $x1 = (ord($char{2}) & 0x0000003F) << 6;
- $x2 = (ord($char{3}) & 0x0000003F);
- return ($z1 | $z2 | $x1 | $x2);
- }
- }
-
- /**
- * utf8-safe fast character iterator
- *
- * Will get the next character starting from $counter, which will then be
- * incremented. If a multi-byte char the bytes will be concatenated and
- * $counter will be incremeted by the number of bytes in the char.
- *
- * @param string $str the string being iterated over
- * @param int &$counter the iterator, will increment by reference
- * @param bool $special_convert whether to do special conversions
- *
- * @return char the next (possibly multi-byte) char from $counter
- * @access private
- */
- static function _next_char($str, &$counter, $special_convert = false)
- {
- $char = $str{$counter++};
- $ord = ord($char);
-
- // for a description of the utf8 system see
- // http://www.phpclasses.org/browse/file/5131.html
-
- // normal ascii one byte char
- if ($ord <= 127) {
- // special conversions needed for this package
- // (that only apply to regular ascii characters)
- // lower case, and convert all non-alphanumeric characters
- // other than "'" to space
- if ($special_convert && $char != ' ' && $char != "'") {
- if ($ord >= 65 && $ord <= 90) { // A-Z
- $char = chr($ord + 32); // lower case
- } elseif ($ord < 97 || $ord > 122) { // NOT a-z
- $char = ' '; // convert to space
- }
- }
-
- return $char;
-
- } elseif ($ord >> 5 == 6) { // two-byte char
- // multi-byte chars
- $nextchar = $str{$counter++}; // get next byte
-
- // lower-casing of non-ascii characters is still incomplete
-
- if ($special_convert) {
- // lower case latin accented characters
- if ($ord == 195) {
- $nextord = ord($nextchar);
- $nextord_adj = $nextord + 64;
- // for a reference, see
- // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
-
- // À - Þ but not ×
- if ($nextord_adj >= 192
- && $nextord_adj <= 222
- && $nextord_adj != 215
- ) {
- $nextchar = chr($nextord + 32);
- }
-
- } elseif ($ord == 208) {
- // lower case cyrillic alphabet
- $nextord = ord($nextchar);
- // if A - Pe
- if ($nextord >= 144 && $nextord <= 159) {
- // lower case
- $nextchar = chr($nextord + 32);
-
- } elseif ($nextord >= 160 && $nextord <= 175) {
- // if Er - Ya
- // lower case
- $char = chr(209); // == $ord++
- $nextchar = chr($nextord - 32);
- }
- }
- }
-
- // tag on next byte
- return $char . $nextchar;
- } elseif ($ord >> 4 == 14) { // three-byte char
-
- // tag on next 2 bytes
- return $char . $str{$counter++} . $str{$counter++};
-
- } elseif ($ord >> 3 == 30) { // four-byte char
-
- // tag on next 3 bytes
- return $char . $str{$counter++} . $str{$counter++} . $str{$counter++};
-
- } else {
- // error?
- }
- }
-
- /**
- * Converts an $language input parameter from the configured mode
- * to the language name that is used internally.
- *
- * Works for strings and arrays.
- *
- * @param string|array $lang A language description ("english"/"en"/"eng")
- * @param boolean $convertKey If $lang is an array, setting $key
- * converts the keys to the language name.
- *
- * @return string|array Language name
- */
- function _convertFromNameMode($lang, $convertKey = false)
- {
- if ($this->_name_mode == 0) {
- return $lang;
- }
-
- if ($this->_name_mode == 2) {
- $method = 'code2ToName';
- } else {
- $method = 'code3ToName';
- }
-
- if (is_string($lang)) {
- return (string)Text_LanguageDetect_ISO639::$method($lang);
- }
-
- $newlang = array();
- foreach ($lang as $key => $val) {
- if ($convertKey) {
- $newkey = (string)Text_LanguageDetect_ISO639::$method($key);
- $newlang[$newkey] = $val;
- } else {
- $newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val);
- }
- }
- return $newlang;
- }
-
- /**
- * Converts an $language output parameter from the language name that is
- * used internally to the configured mode.
- *
- * Works for strings and arrays.
- *
- * @param string|array $lang A language description ("english"/"en"/"eng")
- * @param boolean $convertKey If $lang is an array, setting $key
- * converts the keys to the language name.
- *
- * @return string|array Language name
- */
- function _convertToNameMode($lang, $convertKey = false)
- {
- if ($this->_name_mode == 0) {
- return $lang;
- }
-
- if ($this->_name_mode == 2) {
- $method = 'nameToCode2';
- } else {
- $method = 'nameToCode3';
- }
-
- if (is_string($lang)) {
- return Text_LanguageDetect_ISO639::$method($lang);
- }
-
- $newlang = array();
- foreach ($lang as $key => $val) {
- if ($convertKey) {
- $newkey = Text_LanguageDetect_ISO639::$method($key);
- $newlang[$newkey] = $val;
- } else {
- $newlang[$key] = Text_LanguageDetect_ISO639::$method($val);
- }
- }
- return $newlang;
- }
-}
-
-/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
--- /dev/null
+<?php
+/**
+ * Part of Text_LanguageDetect
+ *
+ * PHP version 5
+ *
+ * @category Text
+ * @package Text_LanguageDetect
+ * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
+ * @copyright 2005-2006 Nicholas Pisarro
+ * @license BSD http://www.opensource.org/licenses/bsd-license.php
+ * @link http://pear.php.net/package/Text_LanguageDetect/
+ */
+
+require_once __DIR__ . '/LanguageDetect/Exception.php';
+require_once __DIR__ . '/LanguageDetect/Parser.php';
+require_once __DIR__ . '/LanguageDetect/ISO639.php';
+
+/**
+ * Detects the language of a given piece of text.
+ *
+ * Attempts to detect the language of a sample of text by correlating ranked
+ * 3-gram frequencies to a table of 3-gram frequencies of known languages.
+ *
+ * Implements a version of a technique originally proposed by Cavnar & Trenkle
+ * (1994): "N-Gram-Based Text Categorization"
+ *
+ * Requires the language model database (lang.dat) that should have
+ * accompanied this class definition in order to be instantiated.
+ *
+ * Example usage:
+ *
+ * <code>
+ * require_once 'Text/LanguageDetect.php';
+ *
+ * $l = new Text_LanguageDetect;
+ *
+ * $stdin = fopen('php://stdin', 'r');
+ *
+ * echo "Supported languages:\n";
+ *
+ * try {
+ * $langs = $l->getLanguages();
+ * } catch (Text_LanguageDetect_Exception $e) {
+ * die($e->getMessage());
+ * }
+ *
+ * sort($langs);
+ * echo join(', ', $langs);
+ *
+ * while ($line = fgets($stdin)) {
+ * print_r($l->detect($line, 4));
+ * }
+ * </code>
+ *
+ * @category Text
+ * @package Text_LanguageDetect
+ * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
+ * @copyright 2005 Nicholas Pisarro
+ * @license BSD http://www.opensource.org/licenses/bsd-license.php
+ * @version Release: @package_version@
+ * @link http://pear.php.net/package/Text_LanguageDetect/
+ *
+ * @SuppressWarnings(PHPMD)
+ */
+class Text_LanguageDetect
+{
+ /**
+ * The filename that stores the trigram data for the detector
+ *
+ * If this value starts with a slash (/) or a dot (.) the value of
+ * $this->_data_dir will be ignored
+ *
+ * @var string
+ */
+ protected $_db_filename = 'lang.dat';
+
+ /**
+ * The filename that stores the unicode block definitions
+ *
+ * If this value starts with a slash (/) or a dot (.) the value of
+ * $this->_data_dir will be ignored
+ *
+ * @var string
+ */
+ protected $_unicode_db_filename = 'unicode_blocks.dat';
+
+ /**
+ * The data directory
+ *
+ * Should be set by PEAR installer
+ *
+ * @var string
+ */
+ protected $_data_dir = '@data_dir@';
+
+ /**
+ * The trigram data for comparison
+ *
+ * Will be loaded on start from $this->_db_filename
+ *
+ * @var array
+ */
+ protected $_lang_db = array();
+
+ /**
+ * Stores the map of the trigram data to unicode characters
+ *
+ * @var array
+ */
+ protected $_unicode_map;
+
+ /**
+ * The size of the trigram data arrays
+ *
+ * @var int
+ */
+ protected $_threshold = 300;
+
+ /**
+ * The maximum possible score.
+ *
+ * Needed for score normalization. Different depending on the
+ * perl compatibility setting
+ *
+ * @var int
+ * @see setPerlCompatible()
+ */
+ protected $_max_score = 0;
+
+ /**
+ * Whether or not to simulate perl's Language::Guess exactly
+ *
+ * @var bool
+ * @see setPerlCompatible()
+ */
+ protected $_perl_compatible = false;
+
+ /**
+ * Whether to use the unicode block detection to speed up processing
+ *
+ * @var bool
+ */
+ protected $_use_unicode_narrowing = true;
+
+ /**
+ * Stores the result of the clustering operation
+ *
+ * @var array
+ * @see clusterLanguages()
+ */
+ protected $_clusters;
+
+ /**
+ * Which type of "language names" are accepted and returned:
+ *
+ * 0 - language name ("english")
+ * 2 - 2-letter ISO 639-1 code ("en")
+ * 3 - 3-letter ISO 639-2 code ("eng")
+ */
+ protected $_name_mode = 0;
+
+ /**
+ * Constructor
+ *
+ * Will attempt to load the language database. If it fails, you will get
+ * an exception.
+ */
+ public function __construct()
+ {
+ $data = $this->_readdb($this->_db_filename);
+ $this->_checkTrigram($data['trigram']);
+ $this->_lang_db = $data['trigram'];
+
+ if (isset($data['trigram-unicodemap'])) {
+ $this->_unicode_map = $data['trigram-unicodemap'];
+ }
+
+ // Not yet implemented:
+ if (isset($data['trigram-clusters'])) {
+ $this->_clusters = $data['trigram-clusters'];
+ }
+ }
+
+ /**
+ * Returns the path to the location of the database
+ *
+ * @param string $fname File name to load
+ *
+ * @return string expected path to the language model database
+ */
+ protected function _get_data_loc($fname)
+ {
+ if ($fname{0} == '/' || $fname{0} == '.') {
+ // if filename starts with a slash, assume it's an absolute pathname
+ // and skip whatever is in $this->_data_dir
+ return $fname;
+
+ } elseif ($this->_data_dir != '@' . 'data_dir' . '@') {
+ // if the data dir was set by the PEAR installer, use that
+ return $this->_data_dir . '/Text_LanguageDetect/' . $fname;
+
+ } else {
+ // assume this was just unpacked somewhere
+ // try the local working directory if otherwise
+ return __DIR__ . '/../data/' . $fname;
+ }
+ }
+
+ /**
+ * Loads the language trigram database from filename
+ *
+ * Trigram datbase should be a serialize()'d array
+ *
+ * @param string $fname the filename where the data is stored
+ *
+ * @return array the language model data
+ * @throws Text_LanguageDetect_Exception
+ */
+ protected function _readdb($fname)
+ {
+ // finds the correct data dir
+ $fname = $this->_get_data_loc($fname);
+
+ // input check
+ if (!file_exists($fname)) {
+ throw new Text_LanguageDetect_Exception(
+ 'Language database does not exist: ' . $fname,
+ Text_LanguageDetect_Exception::DB_NOT_FOUND
+ );
+ } elseif (!is_readable($fname)) {
+ throw new Text_LanguageDetect_Exception(
+ 'Language database is not readable: ' . $fname,
+ Text_LanguageDetect_Exception::DB_NOT_READABLE
+ );
+ }
+
+ return unserialize(file_get_contents($fname));
+ }
+
+
+ /**
+ * Checks if this object is ready to detect languages
+ *
+ * @param array $trigram Trigram data from database
+ *
+ * @return void
+ */
+ protected function _checkTrigram($trigram)
+ {
+ if (!is_array($trigram)) {
+ if (ini_get('magic_quotes_runtime')) {
+ throw new Text_LanguageDetect_Exception(
+ 'Error loading database. Try turning magic_quotes_runtime off.',
+ Text_LanguageDetect_Exception::MAGIC_QUOTES
+ );
+ }
+ throw new Text_LanguageDetect_Exception(
+ 'Language database is not an array.',
+ Text_LanguageDetect_Exception::DB_NOT_ARRAY
+ );
+ } elseif (empty($trigram)) {
+ throw new Text_LanguageDetect_Exception(
+ 'Language database has no elements.',
+ Text_LanguageDetect_Exception::DB_EMPTY
+ );
+ }
+ }
+
+ /**
+ * Omits languages
+ *
+ * Pass this function the name of or an array of names of
+ * languages that you don't want considered
+ *
+ * If you're only expecting a limited set of languages, this can greatly
+ * speed up processing
+ *
+ * @param mixed $omit_list language name or array of names to omit
+ * @param bool $include_only if true will include (rather than
+ * exclude) only those in the list
+ *
+ * @return int number of languages successfully deleted
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function omitLanguages($omit_list, $include_only = false)
+ {
+ $deleted = 0;
+
+ $omit_list = $this->_convertFromNameMode($omit_list);
+
+ if (!$include_only) {
+ // deleting the given languages
+ if (!is_array($omit_list)) {
+ $omit_list = strtolower($omit_list); // case desensitize
+ if (isset($this->_lang_db[$omit_list])) {
+ unset($this->_lang_db[$omit_list]);
+ $deleted++;
+ }
+ } else {
+ foreach ($omit_list as $omit_lang) {
+ if (isset($this->_lang_db[$omit_lang])) {
+ unset($this->_lang_db[$omit_lang]);
+ $deleted++;
+ }
+ }
+ }
+
+ } else {
+ // deleting all except the given languages
+ if (!is_array($omit_list)) {
+ $omit_list = array($omit_list);
+ }
+
+ // case desensitize
+ foreach ($omit_list as $key => $omit_lang) {
+ $omit_list[$key] = strtolower($omit_lang);
+ }
+
+ foreach (array_keys($this->_lang_db) as $lang) {
+ if (!in_array($lang, $omit_list)) {
+ unset($this->_lang_db[$lang]);
+ $deleted++;
+ }
+ }
+ }
+
+ // reset the cluster cache if the number of languages changes
+ // this will then have to be recalculated
+ if (isset($this->_clusters) && $deleted > 0) {
+ $this->_clusters = null;
+ }
+
+ return $deleted;
+ }
+
+
+ /**
+ * Returns the number of languages that this object can detect
+ *
+ * @return int the number of languages
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function getLanguageCount()
+ {
+ return count($this->_lang_db);
+ }
+
+ /**
+ * Checks if the language with the given name exists in the database
+ *
+ * @param mixed $lang Language name or array of language names
+ *
+ * @return bool true if language model exists
+ */
+ public function languageExists($lang)
+ {
+ $lang = $this->_convertFromNameMode($lang);
+
+ if (is_string($lang)) {
+ return isset($this->_lang_db[strtolower($lang)]);
+
+ } elseif (is_array($lang)) {
+ foreach ($lang as $test_lang) {
+ if (!isset($this->_lang_db[strtolower($test_lang)])) {
+ return false;
+ }
+ }
+ return true;
+
+ } else {
+ throw new Text_LanguageDetect_Exception(
+ 'Unsupported parameter type passed to languageExists()',
+ Text_LanguageDetect_Exception::PARAM_TYPE
+ );
+ }
+ }
+
+ /**
+ * Returns the list of detectable languages
+ *
+ * @return array the names of the languages known to this object<<<<<<<
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function getLanguages()
+ {
+ return $this->_convertToNameMode(
+ array_keys($this->_lang_db)
+ );
+ }
+
+ /**
+ * Make this object behave like Language::Guess
+ *
+ * @param bool $setting false to turn off perl compatibility
+ *
+ * @return void
+ */
+ public function setPerlCompatible($setting = true)
+ {
+ if (is_bool($setting)) { // input check
+ $this->_perl_compatible = $setting;
+
+ if ($setting == true) {
+ $this->_max_score = $this->_threshold;
+ } else {
+ $this->_max_score = 0;
+ }
+ }
+
+ }
+
+ /**
+ * Sets the way how language names are accepted and returned.
+ *
+ * @param integer $name_mode One of the following modes:
+ * 0 - language name ("english")
+ * 2 - 2-letter ISO 639-1 code ("en")
+ * 3 - 3-letter ISO 639-2 code ("eng")
+ *
+ * @return void
+ */
+ public function setNameMode($name_mode)
+ {
+ $this->_name_mode = $name_mode;
+ }
+
+ /**
+ * Whether to use unicode block ranges in detection
+ *
+ * Should speed up most detections if turned on (detault is on). In some
+ * circumstances it may be slower, such as for large text samples (> 10K)
+ * in languages that use latin scripts. In other cases it should speed up
+ * detection noticeably.
+ *
+ * @param bool $setting false to turn off
+ *
+ * @return void
+ */
+ public function useUnicodeBlocks($setting = true)
+ {
+ if (is_bool($setting)) {
+ $this->_use_unicode_narrowing = $setting;
+ }
+ }
+
+ /**
+ * Converts a piece of text into trigrams
+ *
+ * @param string $text text to convert
+ *
+ * @return array array of trigram frequencies
+ * @deprecated Superceded by the Text_LanguageDetect_Parser class
+ */
+ protected function _trigram($text)
+ {
+ $s = new Text_LanguageDetect_Parser($text);
+ $s->prepareTrigram();
+ $s->prepareUnicode(false);
+ $s->setPadStart(!$this->_perl_compatible);
+ $s->analyze();
+ return $s->getTrigramFreqs();
+ }
+
+ /**
+ * Converts a set of trigrams from frequencies to ranks
+ *
+ * Thresholds (cuts off) the list at $this->_threshold
+ *
+ * @param array $arr array of trigram
+ *
+ * @return array ranks of trigrams
+ */
+ protected function _arr_rank($arr)
+ {
+
+ // sorts alphabetically first as a standard way of breaking rank ties
+ $this->_bub_sort($arr);
+
+ // below might also work, but seemed to introduce errors in testing
+ //ksort($arr);
+ //asort($arr);
+
+ $rank = array();
+
+ $i = 0;
+ foreach ($arr as $key => $value) {
+ $rank[$key] = $i++;
+
+ // cut off at a standard threshold
+ if ($i >= $this->_threshold) {
+ break;
+ }
+ }
+
+ return $rank;
+ }
+
+ /**
+ * Sorts an array by value breaking ties alphabetically
+ *
+ * @param array $arr the array to sort
+ *
+ * @return void
+ */
+ protected function _bub_sort(&$arr)
+ {
+ // should do the same as this perl statement:
+ // sort { $trigrams{$b} == $trigrams{$a}
+ // ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
+
+ // needs to sort by both key and value at once
+ // using the key to break ties for the value
+
+ // converts array into an array of arrays of each key and value
+ // may be a better way of doing this
+ $combined = array();
+
+ foreach ($arr as $key => $value) {
+ $combined[] = array($key, $value);
+ }
+
+ usort($combined, array($this, '_sort_func'));
+
+ $replacement = array();
+ foreach ($combined as $key => $value) {
+ list($new_key, $new_value) = $value;
+ $replacement[$new_key] = $new_value;
+ }
+
+ $arr = $replacement;
+ }
+
+ /**
+ * Sort function used by bubble sort
+ *
+ * Callback function for usort().
+ *
+ * @param array $a first param passed by usort()
+ * @param array $b second param passed by usort()
+ *
+ * @return int 1 if $a is greater, -1 if not
+ * @see _bub_sort()
+ */
+ protected function _sort_func($a, $b)
+ {
+ // each is actually a key/value pair, so that it can compare using both
+ list($a_key, $a_value) = $a;
+ list($b_key, $b_value) = $b;
+
+ if ($a_value == $b_value) {
+ // if the values are the same, break ties using the key
+ return strcmp($a_key, $b_key);
+
+ } else {
+ // if not, just sort normally
+ if ($a_value > $b_value) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+
+ // 0 should not be possible because keys must be unique
+ }
+
+ /**
+ * Calculates a linear rank-order distance statistic between two sets of
+ * ranked trigrams
+ *
+ * Sums the differences in rank for each trigram. If the trigram does not
+ * appear in both, consider it a difference of $this->_threshold.
+ *
+ * This distance measure was proposed by Cavnar & Trenkle (1994). Despite
+ * its simplicity it has been shown to be highly accurate for language
+ * identification tasks.
+ *
+ * @param array $arr1 the reference set of trigram ranks
+ * @param array $arr2 the target set of trigram ranks
+ *
+ * @return int the sum of the differences between the ranks of
+ * the two trigram sets
+ */
+ protected function _distance($arr1, $arr2)
+ {
+ $sumdist = 0;
+
+ foreach ($arr2 as $key => $value) {
+ if (isset($arr1[$key])) {
+ $distance = abs($value - $arr1[$key]);
+ } else {
+ // $this->_threshold sets the maximum possible distance value
+ // for any one pair of trigrams
+ $distance = $this->_threshold;
+ }
+ $sumdist += $distance;
+ }
+
+ return $sumdist;
+
+ // todo: there are other distance statistics to try, e.g. relative
+ // entropy, but they're probably more costly to compute
+ }
+
+ /**
+ * Normalizes the score returned by _distance()
+ *
+ * Different if perl compatible or not
+ *
+ * @param int $score the score from _distance()
+ * @param int $base_count the number of trigrams being considered
+ *
+ * @return float the normalized score
+ * @see _distance()
+ */
+ protected function _normalize_score($score, $base_count = null)
+ {
+ if ($base_count === null) {
+ $base_count = $this->_threshold;
+ }
+
+ if (!$this->_perl_compatible) {
+ return 1 - ($score / $base_count / $this->_threshold);
+ } else {
+ return floor($score / $base_count);
+ }
+ }
+
+
+ /**
+ * Detects the closeness of a sample of text to the known languages
+ *
+ * Calculates the statistical difference between the text and
+ * the trigrams for each language, normalizes the score then
+ * returns results for all languages in sorted order
+ *
+ * If perl compatible, the score is 300-0, 0 being most similar.
+ * Otherwise, it's 0-1 with 1 being most similar.
+ *
+ * The $sample text should be at least a few sentences in length;
+ * should be ascii-7 or utf8 encoded, if another and the mbstring extension
+ * is present it will try to detect and convert. However, experience has
+ * shown that mb_detect_encoding() *does not work very well* with at least
+ * some types of encoding.
+ *
+ * @param string $sample a sample of text to compare.
+ * @param int $limit if specified, return an array of the most likely
+ * $limit languages and their scores.
+ *
+ * @return mixed sorted array of language scores, blank array if no
+ * useable text was found
+ * @see _distance()
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function detect($sample, $limit = 0)
+ {
+ // input check
+ if (!Text_LanguageDetect_Parser::validateString($sample)) {
+ return array();
+ }
+
+ // check char encoding
+ // (only if mbstring extension is compiled and PHP > 4.0.6)
+ if (function_exists('mb_detect_encoding')
+ && function_exists('mb_convert_encoding')
+ ) {
+ // mb_detect_encoding isn't very reliable, to say the least
+ // detection should still work with a sufficient sample
+ // of ascii characters
+ $encoding = mb_detect_encoding($sample);
+
+ // mb_detect_encoding() will return FALSE if detection fails
+ // don't attempt conversion if that's the case
+ if ($encoding != 'ASCII' && $encoding != 'UTF-8'
+ && $encoding !== false
+ ) {
+ // verify the encoding exists in mb_list_encodings
+ if (in_array($encoding, mb_list_encodings())) {
+ $sample = mb_convert_encoding($sample, 'UTF-8', $encoding);
+ }
+ }
+ }
+
+ $sample_obj = new Text_LanguageDetect_Parser($sample);
+ $sample_obj->prepareTrigram();
+ if ($this->_use_unicode_narrowing) {
+ $sample_obj->prepareUnicode();
+ }
+ $sample_obj->setPadStart(!$this->_perl_compatible);
+ $sample_obj->analyze();
+
+ $trigram_freqs = $sample_obj->getTrigramRanks();
+ $trigram_count = count($trigram_freqs);
+
+ if ($trigram_count == 0) {
+ return array();
+ }
+
+ $scores = array();
+
+ // use unicode block detection to narrow down the possibilities
+ if ($this->_use_unicode_narrowing) {
+ $blocks = $sample_obj->getUnicodeBlocks();
+
+ if (is_array($blocks)) {
+ $present_blocks = array_keys($blocks);
+ } else {
+ throw new Text_LanguageDetect_Exception(
+ 'Error during block detection',
+ Text_LanguageDetect_Exception::BLOCK_DETECTION
+ );
+ }
+
+ $possible_langs = array();
+
+ foreach ($present_blocks as $blockname) {
+ if (isset($this->_unicode_map[$blockname])) {
+
+ $possible_langs = array_merge(
+ $possible_langs,
+ array_keys($this->_unicode_map[$blockname])
+ );
+
+ // todo: faster way to do this?
+ }
+ }
+
+ // could also try an intersect operation rather than a union
+ // in other words, choose languages whose trigrams contain
+ // ALL of the unicode blocks found in this sample
+ // would improve speed but would be completely thrown off by an
+ // unexpected character, like an umlaut appearing in english text
+
+ $possible_langs = array_intersect(
+ array_keys($this->_lang_db),
+ array_unique($possible_langs)
+ );
+
+ // needs to intersect it with the keys of _lang_db in case
+ // languages have been omitted
+
+ } else {
+ // or just try 'em all
+ $possible_langs = array_keys($this->_lang_db);
+ }
+
+
+ foreach ($possible_langs as $lang) {
+ $scores[$lang] = $this->_normalize_score(
+ $this->_distance($this->_lang_db[$lang], $trigram_freqs),
+ $trigram_count
+ );
+ }
+
+ unset($sample_obj);
+
+ if ($this->_perl_compatible) {
+ asort($scores);
+ } else {
+ arsort($scores);
+ }
+
+ // todo: drop languages with a score of $this->_max_score?
+
+ // limit the number of returned scores
+ if ($limit && is_numeric($limit)) {
+ $limited_scores = array();
+
+ $i = 0;
+ foreach ($scores as $key => $value) {
+ if ($i++ >= $limit) {
+ break;
+ }
+
+ $limited_scores[$key] = $value;
+ }
+
+ return $this->_convertToNameMode($limited_scores, true);
+ } else {
+ return $this->_convertToNameMode($scores, true);
+ }
+ }
+
+ /**
+ * Returns only the most similar language to the text sample
+ *
+ * Calls $this->detect() and returns only the top result
+ *
+ * @param string $sample text to detect the language of
+ *
+ * @return string the name of the most likely language
+ * or null if no language is similar
+ * @see detect()
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function detectSimple($sample)
+ {
+ $scores = $this->detect($sample, 1);
+
+ // if top language has the maximum possible score,
+ // then the top score will have been picked at random
+ if (!is_array($scores) || empty($scores)
+ || current($scores) == $this->_max_score
+ ) {
+ return null;
+ } else {
+ return key($scores);
+ }
+ }
+
+ /**
+ * Returns an array containing the most similar language and a confidence
+ * rating
+ *
+ * Confidence is a simple measure calculated from the similarity score
+ * minus the similarity score from the next most similar language
+ * divided by the highest possible score. Languages that have closely
+ * related cousins (e.g. Norwegian and Danish) should generally have lower
+ * confidence scores.
+ *
+ * The similarity score answers the question "How likely is the text the
+ * returned language regardless of the other languages considered?" The
+ * confidence score is one way of answering the question "how likely is the
+ * text the detected language relative to the rest of the language model
+ * set?"
+ *
+ * To see how similar languages are a priori, see languageSimilarity()
+ *
+ * @param string $sample text for which language will be detected
+ *
+ * @return array most similar language, score and confidence rating
+ * or null if no language is similar
+ * @see detect()
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function detectConfidence($sample)
+ {
+ $scores = $this->detect($sample, 2);
+
+ // if most similar language has the max score, it
+ // will have been picked at random
+ if (!is_array($scores) || empty($scores)
+ || current($scores) == $this->_max_score
+ ) {
+ return null;
+ }
+
+ $arr['language'] = key($scores);
+ $arr['similarity'] = current($scores);
+ if (next($scores) !== false) { // if false then no next element
+ // the goal is to return a higher value if the distance between
+ // the similarity of the first score and the second score is high
+
+ if ($this->_perl_compatible) {
+ $arr['confidence'] = (current($scores) - $arr['similarity'])
+ / $this->_max_score;
+
+ } else {
+ $arr['confidence'] = $arr['similarity'] - current($scores);
+
+ }
+
+ } else {
+ $arr['confidence'] = null;
+ }
+
+ return $arr;
+ }
+
+ /**
+ * Returns the distribution of unicode blocks in a given utf8 string
+ *
+ * For the block name of a single char, use unicodeBlockName()
+ *
+ * @param string $str input string. Must be ascii or utf8
+ * @param bool $skip_symbols if true, skip ascii digits, symbols and
+ * non-printing characters. Includes spaces,
+ * newlines and common punctutation characters.
+ *
+ * @return array
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function detectUnicodeBlocks($str, $skip_symbols)
+ {
+ $skip_symbols = (bool)$skip_symbols;
+ $str = (string)$str;
+
+ $sample_obj = new Text_LanguageDetect_Parser($str);
+ $sample_obj->prepareUnicode();
+ $sample_obj->prepareTrigram(false);
+ $sample_obj->setUnicodeSkipSymbols($skip_symbols);
+ $sample_obj->analyze();
+ $blocks = $sample_obj->getUnicodeBlocks();
+ unset($sample_obj);
+ return $blocks;
+ }
+
+ /**
+ * Returns the block name for a given unicode value
+ *
+ * If passed a string, will assume it is being passed a UTF8-formatted
+ * character and will automatically convert. Otherwise it will assume it
+ * is being passed a numeric unicode value.
+ *
+ * Make sure input is of the correct type!
+ *
+ * @param mixed $unicode unicode value or utf8 char
+ *
+ * @return mixed the block name string or false if not found
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function unicodeBlockName($unicode)
+ {
+ if (is_string($unicode)) {
+ // assume it is being passed a utf8 char, so convert it
+ if (self::utf8strlen($unicode) > 1) {
+ throw new Text_LanguageDetect_Exception(
+ 'Pass a single char only to this method',
+ Text_LanguageDetect_Exception::PARAM_TYPE
+ );
+ }
+ $unicode = $this->_utf8char2unicode($unicode);
+
+ } elseif (!is_int($unicode)) {
+ throw new Text_LanguageDetect_Exception(
+ 'Input must be of type string or int.',
+ Text_LanguageDetect_Exception::PARAM_TYPE
+ );
+ }
+
+ $blocks = $this->_read_unicode_block_db();
+
+ $result = $this->_unicode_block_name($unicode, $blocks);
+
+ if ($result == -1) {
+ return false;
+ } else {
+ return $result[2];
+ }
+ }
+
+ /**
+ * Searches the unicode block database
+ *
+ * Returns the block name for a given unicode value. unicodeBlockName() is
+ * the public interface for this function, which does input checks which
+ * this function omits for speed.
+ *
+ * @param int $unicode the unicode value
+ * @param array $blocks the block database
+ * @param int $block_count the number of defined blocks in the database
+ *
+ * @return mixed Block name, -1 if it failed
+ * @see unicodeBlockName()
+ */
+ protected function _unicode_block_name($unicode, $blocks, $block_count = -1)
+ {
+ // for a reference, see
+ // http://www.unicode.org/Public/UNIDATA/Blocks.txt
+
+ // assume that ascii characters are the most common
+ // so try it first for efficiency
+ if ($unicode <= $blocks[0][1]) {
+ return $blocks[0];
+ }
+
+ // the optional $block_count param is for efficiency
+ // so we this function doesn't have to run count() every time
+ if ($block_count != -1) {
+ $high = $block_count - 1;
+ } else {
+ $high = count($blocks) - 1;
+ }
+
+ $low = 1; // start with 1 because ascii was 0
+
+ // your average binary search algorithm
+ while ($low <= $high) {
+ $mid = floor(($low + $high) / 2);
+
+ if ($unicode < $blocks[$mid][0]) {
+ // if it's lower than the lower bound
+ $high = $mid - 1;
+
+ } elseif ($unicode > $blocks[$mid][1]) {
+ // if it's higher than the upper bound
+ $low = $mid + 1;
+
+ } else {
+ // found it
+ return $blocks[$mid];
+ }
+ }
+
+ // failed to find the block
+ return -1;
+
+ // todo: differentiate when it's out of range or when it falls
+ // into an unassigned range?
+ }
+
+ /**
+ * Brings up the unicode block database
+ *
+ * @return array the database of unicode block definitions
+ * @throws Text_LanguageDetect_Exception
+ */
+ protected function _read_unicode_block_db()
+ {
+ // since the unicode definitions are always going to be the same,
+ // might as well share the memory for the db with all other instances
+ // of this class
+ static $data;
+
+ if (!isset($data)) {
+ $data = $this->_readdb($this->_unicode_db_filename);
+ }
+
+ return $data;
+ }
+
+ /**
+ * Calculate the similarities between the language models
+ *
+ * Use this function to see how similar languages are to each other.
+ *
+ * If passed 2 language names, will return just those languages compared.
+ * If passed 1 language name, will return that language compared to
+ * all others.
+ * If passed none, will return an array of every language model compared
+ * to every other one.
+ *
+ * @param string $lang1 the name of the first language to be compared
+ * @param string $lang2 the name of the second language to be compared
+ *
+ * @return array scores of every language compared
+ * or the score of just the provided languages
+ * or null if one of the supplied languages does not exist
+ * @throws Text_LanguageDetect_Exception
+ */
+ public function languageSimilarity($lang1 = null, $lang2 = null)
+ {
+ $lang1 = $this->_convertFromNameMode($lang1);
+ $lang2 = $this->_convertFromNameMode($lang2);
+ if ($lang1 != null) {
+ $lang1 = strtolower($lang1);
+
+ // check if language model exists
+ if (!isset($this->_lang_db[$lang1])) {
+ return null;
+ }
+
+ if ($lang2 != null) {
+ if (!isset($this->_lang_db[$lang2])) {
+ // check if language model exists
+ return null;
+ }
+
+ $lang2 = strtolower($lang2);
+
+ // compare just these two languages
+ return $this->_normalize_score(
+ $this->_distance(
+ $this->_lang_db[$lang1],
+ $this->_lang_db[$lang2]
+ )
+ );
+
+ } else {
+ // compare just $lang1 to all languages
+ $return_arr = array();
+ foreach ($this->_lang_db as $key => $value) {
+ if ($key != $lang1) {
+ // don't compare a language to itself
+ $return_arr[$key] = $this->_normalize_score(
+ $this->_distance($this->_lang_db[$lang1], $value)
+ );
+ }
+ }
+ asort($return_arr);
+
+ return $return_arr;
+ }
+
+
+ } else {
+ // compare all languages to each other
+ $return_arr = array();
+ foreach (array_keys($this->_lang_db) as $lang1) {
+ foreach (array_keys($this->_lang_db) as $lang2) {
+ // skip comparing languages to themselves
+ if ($lang1 != $lang2) {
+
+ if (isset($return_arr[$lang2][$lang1])) {
+ // don't re-calculate what's already been done
+ $return_arr[$lang1][$lang2]
+ = $return_arr[$lang2][$lang1];
+
+ } else {
+ // calculate
+ $return_arr[$lang1][$lang2]
+ = $this->_normalize_score(
+ $this->_distance(
+ $this->_lang_db[$lang1],
+ $this->_lang_db[$lang2]
+ )
+ );
+
+ }
+ }
+ }
+ }
+ return $return_arr;
+ }
+ }
+
+ /**
+ * Cluster known languages according to languageSimilarity()
+ *
+ * WARNING: this method is EXPERIMENTAL. It is not recommended for common
+ * use, and it may disappear or its functionality may change in future
+ * releases without notice.
+ *
+ * Uses a nearest neighbor technique to generate the maximum possible
+ * number of dendograms from the similarity data.
+ *
+ * @return array language cluster data
+ * @throws Text_LanguageDetect_Exception
+ * @see languageSimilarity()
+ * @deprecated this function will eventually be removed and placed into
+ * the model generation class
+ */
+ public function clusterLanguages()
+ {
+ // todo: set the maximum number of clusters
+ // return cached result, if any
+ if (isset($this->_clusters)) {
+ return $this->_clusters;
+ }
+
+ $langs = array_keys($this->_lang_db);
+
+ $arr = $this->languageSimilarity();
+
+ sort($langs);
+
+ foreach ($langs as $lang) {
+ if (!isset($this->_lang_db[$lang])) {
+ throw new Text_LanguageDetect_Exception(
+ "missing $lang!",
+ Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE
+ );
+ }
+ }
+
+ // http://www.psychstat.missouristate.edu/multibook/mlt04m.html
+ foreach ($langs as $old_key => $lang1) {
+ $langs[$lang1] = $lang1;
+ unset($langs[$old_key]);
+ }
+
+ $result_data = $really_map = array();
+
+ $i = 0;
+ while (count($langs) > 2 && $i++ < 200) {
+ $highest_score = -1;
+ $highest_key1 = '';
+ $highest_key2 = '';
+ foreach ($langs as $lang1) {
+ foreach ($langs as $lang2) {
+ if ($lang1 != $lang2
+ && $arr[$lang1][$lang2] > $highest_score
+ ) {
+ $highest_score = $arr[$lang1][$lang2];
+ $highest_key1 = $lang1;
+ $highest_key2 = $lang2;
+ }
+ }
+ }
+
+ if (!$highest_key1) {
+ // should not ever happen
+ throw new Text_LanguageDetect_Exception(
+ "no highest key? (step: $i)",
+ Text_LanguageDetect_Exception::NO_HIGHEST_KEY
+ );
+ }
+
+ if ($highest_score == 0) {
+ // languages are perfectly dissimilar
+ break;
+ }
+
+ // $highest_key1 and $highest_key2 are most similar
+ $sum1 = array_sum($arr[$highest_key1]);
+ $sum2 = array_sum($arr[$highest_key2]);
+
+ // use the score for the one that is most similar to the rest of
+ // the field as the score for the group
+ // todo: could try averaging or "centroid" method instead
+ // seems like that might make more sense
+ // actually nearest neighbor may be better for binary searching
+
+
+ // for "Complete Linkage"/"furthest neighbor"
+ // sign should be <
+ // for "Single Linkage"/"nearest neighbor" method
+ // should should be >
+ // results seem to be pretty much the same with either method
+
+ // figure out which to delete and which to replace
+ if ($sum1 > $sum2) {
+ $replaceme = $highest_key1;
+ $deleteme = $highest_key2;
+ } else {
+ $replaceme = $highest_key2;
+ $deleteme = $highest_key1;
+ }
+
+ $newkey = $replaceme . ':' . $deleteme;
+
+ // $replaceme is most similar to remaining languages
+ // replace $replaceme with '$newkey', deleting $deleteme
+
+ // keep a record of which fork is really which language
+ $really_lang = $replaceme;
+ while (isset($really_map[$really_lang])) {
+ $really_lang = $really_map[$really_lang];
+ }
+ $really_map[$newkey] = $really_lang;
+
+
+ // replace the best fitting key, delete the other
+ foreach ($arr as $key1 => $arr2) {
+ foreach ($arr2 as $key2 => $value2) {
+ if ($key2 == $replaceme) {
+ $arr[$key1][$newkey] = $arr[$key1][$key2];
+ unset($arr[$key1][$key2]);
+ // replacing $arr[$key1][$key2] with $arr[$key1][$newkey]
+ }
+
+ if ($key1 == $replaceme) {
+ $arr[$newkey][$key2] = $arr[$key1][$key2];
+ unset($arr[$key1][$key2]);
+ // replacing $arr[$key1][$key2] with $arr[$newkey][$key2]
+ }
+
+ if ($key1 == $deleteme || $key2 == $deleteme) {
+ // deleting $arr[$key1][$key2]
+ unset($arr[$key1][$key2]);
+ }
+ }
+ }
+
+
+ unset($langs[$highest_key1]);
+ unset($langs[$highest_key2]);
+ $langs[$newkey] = $newkey;
+
+
+ // some of these may be overkill
+ $result_data[$newkey] = array(
+ 'newkey' => $newkey,
+ 'count' => $i,
+ 'diff' => abs($sum1 - $sum2),
+ 'score' => $highest_score,
+ 'bestfit' => $replaceme,
+ 'otherfit' => $deleteme,
+ 'really' => $really_lang,
+ );
+ }
+
+ $return_val = array(
+ 'open_forks' => $langs,
+ // the top level of clusters
+ // clusters that are mutually exclusive
+ // or specified by a specific maximum
+
+ 'fork_data' => $result_data,
+ // data for each split
+
+ 'name_map' => $really_map,
+ // which cluster is really which language
+ // using the nearest neighbor technique, the cluster
+ // inherits all of the properties of its most-similar member
+ // this keeps track
+ );
+
+
+ // saves the result in the object
+ $this->_clusters = $return_val;
+
+ return $return_val;
+ }
+
+
+ /**
+ * Perform an intelligent detection based on clusterLanguages()
+ *
+ * WARNING: this method is EXPERIMENTAL. It is not recommended for common
+ * use, and it may disappear or its functionality may change in future
+ * releases without notice.
+ *
+ * This compares the sample text to top the top level of clusters. If the
+ * sample is similar to the cluster it will drop down and compare it to the
+ * languages in the cluster, and so on until it hits a leaf node.
+ *
+ * this should find the language in considerably fewer compares
+ * (the equivalent of a binary search), however clusterLanguages() is costly
+ * and the loss of accuracy from this technique is significant.
+ *
+ * This method may need to be 'fuzzier' in order to become more accurate.
+ *
+ * This function could be more useful if the universe of possible languages
+ * was very large, however in such cases some method of Bayesian inference
+ * might be more helpful.
+ *
+ * @param string $str input string
+ *
+ * @return array language scores (only those compared)
+ * @throws Text_LanguageDetect_Exception
+ * @see clusterLanguages()
+ */
+ public function clusteredSearch($str)
+ {
+ // input check
+ if (!Text_LanguageDetect_Parser::validateString($str)) {
+ return array();
+ }
+
+ // clusterLanguages() will return a cached result if possible
+ // so it's safe to call it every time
+ $result = $this->clusterLanguages();
+
+ $dendogram_start = $result['open_forks'];
+ $dendogram_data = $result['fork_data'];
+ $dendogram_alias = $result['name_map'];
+
+ $sample_obj = new Text_LanguageDetect_Parser($str);
+ $sample_obj->prepareTrigram();
+ $sample_obj->setPadStart(!$this->_perl_compatible);
+ $sample_obj->analyze();
+ $sample_result = $sample_obj->getTrigramRanks();
+ $sample_count = count($sample_result);
+
+ // input check
+ if ($sample_count == 0) {
+ return array();
+ }
+
+ $i = 0; // counts the number of steps
+
+ foreach ($dendogram_start as $lang) {
+ if (isset($dendogram_alias[$lang])) {
+ $lang_key = $dendogram_alias[$lang];
+ } else {
+ $lang_key = $lang;
+ }
+
+ $scores[$lang] = $this->_normalize_score(
+ $this->_distance($this->_lang_db[$lang_key], $sample_result),
+ $sample_count
+ );
+
+ $i++;
+ }
+
+ if ($this->_perl_compatible) {
+ asort($scores);
+ } else {
+ arsort($scores);
+ }
+
+ $top_score = current($scores);
+ $top_key = key($scores);
+
+ // of starting forks, $top_key is the most similar to the sample
+
+ $cur_key = $top_key;
+ while (isset($dendogram_data[$cur_key])) {
+ $lang1 = $dendogram_data[$cur_key]['bestfit'];
+ $lang2 = $dendogram_data[$cur_key]['otherfit'];
+ foreach (array($lang1, $lang2) as $lang) {
+ if (isset($dendogram_alias[$lang])) {
+ $lang_key = $dendogram_alias[$lang];
+ } else {
+ $lang_key = $lang;
+ }
+
+ $scores[$lang] = $this->_normalize_score(
+ $this->_distance($this->_lang_db[$lang_key], $sample_result),
+ $sample_count
+ );
+
+ //todo: does not need to do same comparison again
+ }
+
+ $i++;
+
+ if ($scores[$lang1] > $scores[$lang2]) {
+ $cur_key = $lang1;
+ $loser_key = $lang2;
+ } else {
+ $cur_key = $lang2;
+ $loser_key = $lang1;
+ }
+
+ $diff = $scores[$cur_key] - $scores[$loser_key];
+
+ // $cur_key ({$dendogram_alias[$cur_key]}) wins
+ // over $loser_key ({$dendogram_alias[$loser_key]})
+ // with a difference of $diff
+ }
+
+ // found result in $i compares
+
+ // rather than sorting the result, preserve it so that you can see
+ // which paths the algorithm decided to take along the tree
+
+ // but sometimes the last item is only the second highest
+ if (($this->_perl_compatible && (end($scores) > prev($scores)))
+ || (!$this->_perl_compatible && (end($scores) < prev($scores)))
+ ) {
+ $real_last_score = current($scores);
+ $real_last_key = key($scores);
+
+ // swaps the 2nd-to-last item for the last item
+ unset($scores[$real_last_key]);
+ $scores[$real_last_key] = $real_last_score;
+ }
+
+
+ if (!$this->_perl_compatible) {
+ $scores = array_reverse($scores, true);
+ // second param requires php > 4.0.3
+ }
+
+ return $scores;
+ }
+
+ /**
+ * UTF8-safe strlen()
+ *
+ * Returns the numbers of characters (not bytes) in a utf8 string
+ *
+ * @param string $str string to get the length of
+ *
+ * @return int number of chars
+ */
+ public static function utf8strlen($str)
+ {
+ // utf8_decode() will convert unknown chars to '?', which is actually
+ // ideal for counting.
+
+ return strlen(utf8_decode($str));
+
+ // idea stolen from dokuwiki
+ }
+
+ /**
+ * Returns the unicode value of a utf8 char
+ *
+ * @param string $char a utf8 (possibly multi-byte) char
+ *
+ * @return int unicode value
+ * @link http://en.wikipedia.org/wiki/UTF-8
+ */
+ protected function _utf8char2unicode($char)
+ {
+ // strlen() here will actually get the binary length of a single char
+ switch (strlen($char)) {
+ case 1:
+ // normal ASCII-7 byte
+ // 0xxxxxxx --> 0xxxxxxx
+ return ord($char{0});
+
+ case 2:
+ // 2 byte unicode
+ // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx
+ $z = (ord($char{0}) & 0x000001F) << 6;
+ $x = (ord($char{1}) & 0x0000003F);
+ return ($z | $x);
+
+ case 3:
+ // 3 byte unicode
+ // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx
+ $z = (ord($char{0}) & 0x0000000F) << 12;
+ $x1 = (ord($char{1}) & 0x0000003F) << 6;
+ $x2 = (ord($char{2}) & 0x0000003F);
+ return ($z | $x1 | $x2);
+
+ case 4:
+ // 4 byte unicode
+ // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx -->
+ // 000zzzzz xxxxxxxx xxxxxxxx
+ $z1 = (ord($char{0}) & 0x00000007) << 18;
+ $z2 = (ord($char{1}) & 0x0000003F) << 12;
+ $x1 = (ord($char{2}) & 0x0000003F) << 6;
+ $x2 = (ord($char{3}) & 0x0000003F);
+ return ($z1 | $z2 | $x1 | $x2);
+ }
+ }
+
+ /**
+ * UTF8-safe fast character iterator
+ *
+ * Will get the next character starting from $counter, which will then be
+ * incremented. If a multi-byte char the bytes will be concatenated and
+ * $counter will be incremeted by the number of bytes in the char.
+ *
+ * @param string $str the string being iterated over
+ * @param int $counter the iterator, will increment by reference
+ * @param bool $special_convert whether to do special conversions
+ *
+ * @return char the next (possibly multi-byte) char from $counter
+ */
+ protected static function _next_char($str, &$counter, $special_convert = false)
+ {
+ $char = $str{$counter++};
+ $ord = ord($char);
+
+ // for a description of the utf8 system see
+ // http://www.phpclasses.org/browse/file/5131.html
+
+ // normal ascii one byte char
+ if ($ord <= 127) {
+ // special conversions needed for this package
+ // (that only apply to regular ascii characters)
+ // lower case, and convert all non-alphanumeric characters
+ // other than "'" to space
+ if ($special_convert && $char != ' ' && $char != "'") {
+ if ($ord >= 65 && $ord <= 90) { // A-Z
+ $char = chr($ord + 32); // lower case
+ } elseif ($ord < 97 || $ord > 122) { // NOT a-z
+ $char = ' '; // convert to space
+ }
+ }
+
+ return $char;
+
+ } elseif ($ord >> 5 == 6) { // two-byte char
+ // multi-byte chars
+ $nextchar = $str{$counter++}; // get next byte
+
+ // lower-casing of non-ascii characters is still incomplete
+
+ if ($special_convert) {
+ // lower case latin accented characters
+ if ($ord == 195) {
+ $nextord = ord($nextchar);
+ $nextord_adj = $nextord + 64;
+ // for a reference, see
+ // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
+
+ // À - Þ but not ×
+ if ($nextord_adj >= 192
+ && $nextord_adj <= 222
+ && $nextord_adj != 215
+ ) {
+ $nextchar = chr($nextord + 32);
+ }
+
+ } elseif ($ord == 208) {
+ // lower case cyrillic alphabet
+ $nextord = ord($nextchar);
+ // if A - Pe
+ if ($nextord >= 144 && $nextord <= 159) {
+ // lower case
+ $nextchar = chr($nextord + 32);
+
+ } elseif ($nextord >= 160 && $nextord <= 175) {
+ // if Er - Ya
+ // lower case
+ $char = chr(209); // == $ord++
+ $nextchar = chr($nextord - 32);
+ }
+ }
+ }
+
+ // tag on next byte
+ return $char . $nextchar;
+ } elseif ($ord >> 4 == 14) { // three-byte char
+
+ // tag on next 2 bytes
+ return $char . $str{$counter++} . $str{$counter++};
+
+ } elseif ($ord >> 3 == 30) { // four-byte char
+
+ // tag on next 3 bytes
+ return $char . $str{$counter++} . $str{$counter++} . $str{$counter++};
+
+ } else {
+ // error?
+ }
+ }
+
+ /**
+ * Converts an $language input parameter from the configured mode
+ * to the language name that is used internally.
+ *
+ * Works for strings and arrays.
+ *
+ * @param string|array $lang A language description ("english"/"en"/"eng")
+ * @param boolean $convertKey If $lang is an array, setting $key
+ * converts the keys to the language name.
+ *
+ * @return string|array Language name
+ */
+ protected function _convertFromNameMode($lang, $convertKey = false)
+ {
+ if ($this->_name_mode == 0) {
+ return $lang;
+ }
+
+ if ($this->_name_mode == 2) {
+ $method = 'code2ToName';
+ } else {
+ $method = 'code3ToName';
+ }
+
+ if (is_string($lang)) {
+ return (string)Text_LanguageDetect_ISO639::$method($lang);
+ }
+
+ $newlang = array();
+ foreach ($lang as $key => $val) {
+ if ($convertKey) {
+ $newkey = (string)Text_LanguageDetect_ISO639::$method($key);
+ $newlang[$newkey] = $val;
+ } else {
+ $newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val);
+ }
+ }
+ return $newlang;
+ }
+
+ /**
+ * Converts an $language output parameter from the language name that is
+ * used internally to the configured mode.
+ *
+ * Works for strings and arrays.
+ *
+ * @param string|array $lang A language description ("english"/"en"/"eng")
+ * @param boolean $convertKey If $lang is an array, setting $key
+ * converts the keys to the language name.
+ *
+ * @return string|array Language name
+ */
+ protected function _convertToNameMode($lang, $convertKey = false)
+ {
+ if ($this->_name_mode == 0) {
+ return $lang;
+ }
+
+ if ($this->_name_mode == 2) {
+ $method = 'nameToCode2';
+ } else {
+ $method = 'nameToCode3';
+ }
+
+ if (is_string($lang)) {
+ return Text_LanguageDetect_ISO639::$method($lang);
+ }
+
+ $newlang = array();
+ foreach ($lang as $key => $val) {
+ if ($convertKey) {
+ $newkey = Text_LanguageDetect_ISO639::$method($key);
+ $newlang[$newkey] = $val;
+ } else {
+ $newlang[$key] = Text_LanguageDetect_ISO639::$method($val);
+ }
+ }
+ return $newlang;
+ }
+}
+
+/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
<?php
+/**
+ * Part of Text_LanguageDetect
+ *
+ * PHP version 5
+ *
+ * @category Text
+ * @package Text_LanguageDetect
+ * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
+ * @license BSD http://www.opensource.org/licenses/bsd-license.php
+ * @link http://pear.php.net/package/Text_LanguageDetect/
+ */
+
+/**
+ * Part of the PEAR language detection package
+ *
+ * PHP version 5
+ *
+ * @category Text
+ * @package Text_LanguageDetect
+ * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
+ * @license BSD http://www.opensource.org/licenses/bsd-license.php
+ * @link http://pear.php.net/package/Text_LanguageDetect/
+ * @link http://langdetect.blogspot.com/
+ */
class Text_LanguageDetect_Exception extends Exception
{
/**
* @author Christian Weiske <cweiske@php.net>
* @copyright 2011 Christian Weiske <cweiske@php.net>
* @license http://www.debian.org/misc/bsd.license BSD
- * @version SVN: $Id$
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
* @package Text_LanguageDetect
* @author Christian Weiske <cweiske@php.net>
* @copyright 2011 Christian Weiske <cweiske@php.net>
- * @license http://www.debian.org/misc/bsd.license BSD
+ * @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://www.loc.gov/standards/iso639-2/php/code_list.php
*
* @SuppressWarnings(PHPMD)
<?php
-
/**
- * This class represents a text sample to be parsed.
+ * Part of Text_LanguageDetect
+ *
+ * PHP version 5
*
- * @category Text
- * @package Text_LanguageDetect
- * @author Nicholas Pisarro
- * @copyright 2006
- * @license BSD
- * @version CVS: $Id: Parser.php 322327 2012-01-15 17:55:59Z cweiske $
- * @link http://pear.php.net/package/Text_LanguageDetect/
- * @link http://langdetect.blogspot.com/
+ * @category Text
+ * @package Text_LanguageDetect
+ * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
+ * @copyright 2006 Nicholas Pisarro
+ * @license BSD http://www.opensource.org/licenses/bsd-license.php
+ * @link http://pear.php.net/package/Text_LanguageDetect/
*/
/**
* class. After a new profile has been built, the data can be retrieved using
* the accessor functions.
*
- * This class is intended to be used by the Text_LanguageDetect class, not
+ * This class is intended to be used by the Text_LanguageDetect class, not
* end-users.
*
- * @category Text
- * @package Text_LanguageDetect
- * @author Nicholas Pisarro
- * @copyright 2006
- * @license BSD
- * @version release: 0.3.0
+ * @category Text
+ * @package Text_LanguageDetect
+ * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
+ * @copyright 2006 Nicholas Pisarro
+ * @license BSD http://www.opensource.org/licenses/bsd-license.php
+ * @version Release: @package_version@
+ * @link http://pear.php.net/package/Text_LanguageDetect/
*/
class Text_LanguageDetect_Parser extends Text_LanguageDetect
{
/**
- * the piece of text being parsed
+ * The piece of text being parsed
*
- * @access private
- * @var string
+ * @var string
*/
- var $_string;
+ protected $_string;
/**
- * stores the trigram frequencies of the sample
+ * Stores the trigram frequencies of the sample
*
- * @access private
- * @var string
+ * @var string
*/
- var $_trigrams = array();
+ protected $_trigrams = array();
/**
- * stores the trigram ranks of the sample
+ * Stores the trigram ranks of the sample
*
- * @access private
- * @var array
+ * @var array
*/
- var $_trigram_ranks = array();
+ protected $_trigram_ranks = array();
/**
- * stores the unicode blocks of the sample
+ * Stores the unicode blocks of the sample
*
- * @access private
- * @var array
+ * @var array
*/
- var $_unicode_blocks = array();
-
+ protected $_unicode_blocks = array();
+
/**
* Whether the parser should compile the unicode ranges
- *
- * @access private
- * @var bool
+ *
+ * @var bool
*/
- var $_compile_unicode = false;
+ protected $_compile_unicode = false;
/**
* Whether the parser should compile trigrams
*
- * @access private
- * @var bool
+ * @var bool
*/
- var $_compile_trigram = false;
+ protected $_compile_trigram = false;
/**
* Whether the trigram parser should pad the beginning of the string
*
- * @access private
- * @var bool
+ * @var bool
*/
- var $_trigram_pad_start = false;
+ protected $_trigram_pad_start = false;
/**
* Whether the unicode parser should skip non-alphabetical ascii chars
*
- * @access private
- * @var bool
+ * @var bool
*/
- var $_unicode_skip_symbols = true;
+ protected $_unicode_skip_symbols = true;
/**
* Constructor
*
- * @access private
- * @param string $string string to be parsed
+ * @param string $string string to be parsed
*/
- function Text_LanguageDetect_Parser($string) {
+ public function __construct($string)
+ {
$this->_string = $string;
}
+ /**
+ * PHP 4 constructor for backwards compatibility.
+ *
+ * @param string $string string to be parsed
+ *
+ * @return void
+ */
+ public function Text_LanguageDetect_Parser($string)
+ {
+ self::__construct($string);
+ }
+
/**
* Returns true if a string is suitable for parsing
*
- * @param string $str input string to test
- * @return bool true if acceptable, false if not
+ * @param string $str input string to test
+ *
+ * @return bool true if acceptable, false if not
*/
- public static function validateString($str) {
+ public static function validateString($str)
+ {
if (!empty($str) && strlen($str) > 3 && preg_match('/\S/', $str)) {
return true;
} else {
}
/**
- * turn on/off trigram counting
+ * Turn on/off trigram counting
*
- * @access public
- * @param bool $bool true for on, false for off
+ * @param bool $bool true for on, false for off
+ *
+ * @return void
*/
- function prepareTrigram($bool = true)
+ public function prepareTrigram($bool = true)
{
$this->_compile_trigram = $bool;
}
/**
- * turn on/off unicode block counting
+ * Turn on/off unicode block counting
+ *
+ * @param bool $bool true for on, false for off
*
- * @access public
- * @param bool $bool true for on, false for off
+ * @return void
*/
- function prepareUnicode($bool = true)
+ public function prepareUnicode($bool = true)
{
$this->_compile_unicode = $bool;
}
/**
- * turn on/off padding the beginning of the sample string
+ * Turn on/off padding the beginning of the sample string
+ *
+ * @param bool $bool true for on, false for off
*
- * @access public
- * @param bool $bool true for on, false for off
+ * @return void
*/
- function setPadStart($bool = true)
+ public function setPadStart($bool = true)
{
$this->_trigram_pad_start = $bool;
}
/**
* Should the unicode block counter skip non-alphabetical ascii chars?
*
- * @access public
- * @param bool $bool true for on, false for off
+ * @param bool $bool true for on, false for off
+ *
+ * @return void
*/
- function setUnicodeSkipSymbols($bool = true)
+ public function setUnicodeSkipSymbols($bool = true)
{
$this->_unicode_skip_symbols = $bool;
}
/**
* Returns the trigram ranks for the text sample
*
- * @access public
- * @return array trigram ranks in the text sample
+ * @return array Trigram ranks in the text sample
*/
- function &getTrigramRanks()
+ public function getTrigramRanks()
{
return $this->_trigram_ranks;
}
/**
* Return the trigram freqency table
*
- * only used in testing to make sure the parser is working
+ * Only used in testing to make sure the parser is working
*
- * @access public
- * @return array trigram freqencies in the text sample
+ * @return array Trigram freqencies in the text sample
*/
- function &getTrigramFreqs()
+ public function getTrigramFreqs()
{
return $this->_trigram;
}
/**
- * returns the array of unicode blocks
+ * Returns the array of unicode blocks
*
- * @access public
- * @return array unicode blocks in the text sample
+ * @return array Unicode blocks in the text sample
*/
- function &getUnicodeBlocks()
+ public function getUnicodeBlocks()
{
return $this->_unicode_blocks;
}
/**
* Executes the parsing operation
- *
- * Be sure to call the set*() functions to set options and the
+ *
+ * Be sure to call the set*() functions to set options and the
* prepare*() functions first to tell it what kind of data to compute
*
* Afterwards the get*() functions can be used to access the compiled
* information.
*
- * @access public
+ * @return void
*/
- function analyze()
+ public function analyze()
{
$len = strlen($this->_string);
$byte_counter = 0;
if ($this->_compile_trigram) {
if (!($b == ' ' && ($a == ' ' || $char == ' '))) {
if (!isset($this->_trigram[$a . $b . $char])) {
- $this->_trigram[$a . $b . $char] = 1;
+ $this->_trigram[$a . $b . $char] = 1;
} else {
- $this->_trigram[$a . $b . $char]++;
+ $this->_trigram[$a . $b . $char]++;
}
}
// unicode block detection
if ($this->_compile_unicode) {
if ($this->_unicode_skip_symbols
- && strlen($char) == 1
- && ($char < 'A' || $char > 'z'
- || ($char > 'Z' && $char < 'a'))
- && $char != "'") { // does not skip the apostrophe
+ && strlen($char) == 1
+ && ($char < 'A' || $char > 'z'
+ || ($char > 'Z' && $char < 'a'))
+ && $char != "'"
+ ) { // does not skip the apostrophe
// since it's included in the language
// models
if ($this->_compile_unicode) {
foreach ($unicode_chars as $utf8_char => $count) {
$search_result = $this->_unicode_block_name(
- $this->_utf8char2unicode($utf8_char), $blocks, $block_count);
+ $this->_utf8char2unicode($utf8_char), $blocks, $block_count
+ );
if ($search_result != -1) {
$block_name = $search_result[2];
--- /dev/null
+<?php
+/**
+ * Generate the serialized unicode_blocks.dat file shipped with the package
+ */
+$unicode_blocks = include __DIR__ . '/unicode_blocks.php';
+file_put_contents(__DIR__ . '/unicode_blocks.dat', serialize($unicode_blocks));
+
-a:145:{i:0;a:3:{i:0;s:6:"0x0000";i:1;s:6:"0x007F";i:2;s:11:"Basic Latin";}i:1;a:3:{i:0;s:6:"0x0080";i:1;s:6:"0x00FF";i:2;s:18:"Latin-1 Supplement";}i:2;a:3:{i:0;s:6:"0x0100";i:1;s:6:"0x017F";i:2;s:16:"Latin Extended-A";}i:3;a:3:{i:0;s:6:"0x0180";i:1;s:6:"0x024F";i:2;s:16:"Latin Extended-B";}i:4;a:3:{i:0;s:6:"0x0250";i:1;s:6:"0x02AF";i:2;s:14:"IPA Extensions";}i:5;a:3:{i:0;s:6:"0x02B0";i:1;s:6:"0x02FF";i:2;s:24:"Spacing Modifier Letters";}i:6;a:3:{i:0;s:6:"0x0300";i:1;s:6:"0x036F";i:2;s:27:"Combining Diacritical Marks";}i:7;a:3:{i:0;s:6:"0x0370";i:1;s:6:"0x03FF";i:2;s:16:"Greek and Coptic";}i:8;a:3:{i:0;s:6:"0x0400";i:1;s:6:"0x04FF";i:2;s:8:"Cyrillic";}i:9;a:3:{i:0;s:6:"0x0500";i:1;s:6:"0x052F";i:2;s:19:"Cyrillic Supplement";}i:10;a:3:{i:0;s:6:"0x0530";i:1;s:6:"0x058F";i:2;s:8:"Armenian";}i:11;a:3:{i:0;s:6:"0x0590";i:1;s:6:"0x05FF";i:2;s:6:"Hebrew";}i:12;a:3:{i:0;s:6:"0x0600";i:1;s:6:"0x06FF";i:2;s:6:"Arabic";}i:13;a:3:{i:0;s:6:"0x0700";i:1;s:6:"0x074F";i:2;s:6:"Syriac";}i:14;a:3:{i:0;s:6:"0x0750";i:1;s:6:"0x077F";i:2;s:17:"Arabic Supplement";}i:15;a:3:{i:0;s:6:"0x0780";i:1;s:6:"0x07BF";i:2;s:6:"Thaana";}i:16;a:3:{i:0;s:6:"0x0900";i:1;s:6:"0x097F";i:2;s:10:"Devanagari";}i:17;a:3:{i:0;s:6:"0x0980";i:1;s:6:"0x09FF";i:2;s:7:"Bengali";}i:18;a:3:{i:0;s:6:"0x0A00";i:1;s:6:"0x0A7F";i:2;s:8:"Gurmukhi";}i:19;a:3:{i:0;s:6:"0x0A80";i:1;s:6:"0x0AFF";i:2;s:8:"Gujarati";}i:20;a:3:{i:0;s:6:"0x0B00";i:1;s:6:"0x0B7F";i:2;s:5:"Oriya";}i:21;a:3:{i:0;s:6:"0x0B80";i:1;s:6:"0x0BFF";i:2;s:5:"Tamil";}i:22;a:3:{i:0;s:6:"0x0C00";i:1;s:6:"0x0C7F";i:2;s:6:"Telugu";}i:23;a:3:{i:0;s:6:"0x0C80";i:1;s:6:"0x0CFF";i:2;s:7:"Kannada";}i:24;a:3:{i:0;s:6:"0x0D00";i:1;s:6:"0x0D7F";i:2;s:9:"Malayalam";}i:25;a:3:{i:0;s:6:"0x0D80";i:1;s:6:"0x0DFF";i:2;s:7:"Sinhala";}i:26;a:3:{i:0;s:6:"0x0E00";i:1;s:6:"0x0E7F";i:2;s:4:"Thai";}i:27;a:3:{i:0;s:6:"0x0E80";i:1;s:6:"0x0EFF";i:2;s:3:"Lao";}i:28;a:3:{i:0;s:6:"0x0F00";i:1;s:6:"0x0FFF";i:2;s:7:"Tibetan";}i:29;a:3:{i:0;s:6:"0x1000";i:1;s:6:"0x109F";i:2;s:7:"Myanmar";}i:30;a:3:{i:0;s:6:"0x10A0";i:1;s:6:"0x10FF";i:2;s:8:"Georgian";}i:31;a:3:{i:0;s:6:"0x1100";i:1;s:6:"0x11FF";i:2;s:11:"Hangul Jamo";}i:32;a:3:{i:0;s:6:"0x1200";i:1;s:6:"0x137F";i:2;s:8:"Ethiopic";}i:33;a:3:{i:0;s:6:"0x1380";i:1;s:6:"0x139F";i:2;s:19:"Ethiopic Supplement";}i:34;a:3:{i:0;s:6:"0x13A0";i:1;s:6:"0x13FF";i:2;s:8:"Cherokee";}i:35;a:3:{i:0;s:6:"0x1400";i:1;s:6:"0x167F";i:2;s:37:"Unified Canadian Aboriginal Syllabics";}i:36;a:3:{i:0;s:6:"0x1680";i:1;s:6:"0x169F";i:2;s:5:"Ogham";}i:37;a:3:{i:0;s:6:"0x16A0";i:1;s:6:"0x16FF";i:2;s:5:"Runic";}i:38;a:3:{i:0;s:6:"0x1700";i:1;s:6:"0x171F";i:2;s:7:"Tagalog";}i:39;a:3:{i:0;s:6:"0x1720";i:1;s:6:"0x173F";i:2;s:7:"Hanunoo";}i:40;a:3:{i:0;s:6:"0x1740";i:1;s:6:"0x175F";i:2;s:5:"Buhid";}i:41;a:3:{i:0;s:6:"0x1760";i:1;s:6:"0x177F";i:2;s:8:"Tagbanwa";}i:42;a:3:{i:0;s:6:"0x1780";i:1;s:6:"0x17FF";i:2;s:5:"Khmer";}i:43;a:3:{i:0;s:6:"0x1800";i:1;s:6:"0x18AF";i:2;s:9:"Mongolian";}i:44;a:3:{i:0;s:6:"0x1900";i:1;s:6:"0x194F";i:2;s:5:"Limbu";}i:45;a:3:{i:0;s:6:"0x1950";i:1;s:6:"0x197F";i:2;s:6:"Tai Le";}i:46;a:3:{i:0;s:6:"0x1980";i:1;s:6:"0x19DF";i:2;s:11:"New Tai Lue";}i:47;a:3:{i:0;s:6:"0x19E0";i:1;s:6:"0x19FF";i:2;s:13:"Khmer Symbols";}i:48;a:3:{i:0;s:6:"0x1A00";i:1;s:6:"0x1A1F";i:2;s:8:"Buginese";}i:49;a:3:{i:0;s:6:"0x1D00";i:1;s:6:"0x1D7F";i:2;s:19:"Phonetic Extensions";}i:50;a:3:{i:0;s:6:"0x1D80";i:1;s:6:"0x1DBF";i:2;s:30:"Phonetic Extensions Supplement";}i:51;a:3:{i:0;s:6:"0x1DC0";i:1;s:6:"0x1DFF";i:2;s:38:"Combining Diacritical Marks Supplement";}i:52;a:3:{i:0;s:6:"0x1E00";i:1;s:6:"0x1EFF";i:2;s:25:"Latin Extended Additional";}i:53;a:3:{i:0;s:6:"0x1F00";i:1;s:6:"0x1FFF";i:2;s:14:"Greek Extended";}i:54;a:3:{i:0;s:6:"0x2000";i:1;s:6:"0x206F";i:2;s:19:"General Punctuation";}i:55;a:3:{i:0;s:6:"0x2070";i:1;s:6:"0x209F";i:2;s:27:"Superscripts and Subscripts";}i:56;a:3:{i:0;s:6:"0x20A0";i:1;s:6:"0x20CF";i:2;s:16:"Currency Symbols";}i:57;a:3:{i:0;s:6:"0x20D0";i:1;s:6:"0x20FF";i:2;s:39:"Combining Diacritical Marks for Symbols";}i:58;a:3:{i:0;s:6:"0x2100";i:1;s:6:"0x214F";i:2;s:18:"Letterlike Symbols";}i:59;a:3:{i:0;s:6:"0x2150";i:1;s:6:"0x218F";i:2;s:12:"Number Forms";}i:60;a:3:{i:0;s:6:"0x2190";i:1;s:6:"0x21FF";i:2;s:6:"Arrows";}i:61;a:3:{i:0;s:6:"0x2200";i:1;s:6:"0x22FF";i:2;s:22:"Mathematical Operators";}i:62;a:3:{i:0;s:6:"0x2300";i:1;s:6:"0x23FF";i:2;s:23:"Miscellaneous Technical";}i:63;a:3:{i:0;s:6:"0x2400";i:1;s:6:"0x243F";i:2;s:16:"Control Pictures";}i:64;a:3:{i:0;s:6:"0x2440";i:1;s:6:"0x245F";i:2;s:29:"Optical Character Recognition";}i:65;a:3:{i:0;s:6:"0x2460";i:1;s:6:"0x24FF";i:2;s:22:"Enclosed Alphanumerics";}i:66;a:3:{i:0;s:6:"0x2500";i:1;s:6:"0x257F";i:2;s:11:"Box Drawing";}i:67;a:3:{i:0;s:6:"0x2580";i:1;s:6:"0x259F";i:2;s:14:"Block Elements";}i:68;a:3:{i:0;s:6:"0x25A0";i:1;s:6:"0x25FF";i:2;s:16:"Geometric Shapes";}i:69;a:3:{i:0;s:6:"0x2600";i:1;s:6:"0x26FF";i:2;s:21:"Miscellaneous Symbols";}i:70;a:3:{i:0;s:6:"0x2700";i:1;s:6:"0x27BF";i:2;s:8:"Dingbats";}i:71;a:3:{i:0;s:6:"0x27C0";i:1;s:6:"0x27EF";i:2;s:36:"Miscellaneous Mathematical Symbols-A";}i:72;a:3:{i:0;s:6:"0x27F0";i:1;s:6:"0x27FF";i:2;s:21:"Supplemental Arrows-A";}i:73;a:3:{i:0;s:6:"0x2800";i:1;s:6:"0x28FF";i:2;s:16:"Braille Patterns";}i:74;a:3:{i:0;s:6:"0x2900";i:1;s:6:"0x297F";i:2;s:21:"Supplemental Arrows-B";}i:75;a:3:{i:0;s:6:"0x2980";i:1;s:6:"0x29FF";i:2;s:36:"Miscellaneous Mathematical Symbols-B";}i:76;a:3:{i:0;s:6:"0x2A00";i:1;s:6:"0x2AFF";i:2;s:35:"Supplemental Mathematical Operators";}i:77;a:3:{i:0;s:6:"0x2B00";i:1;s:6:"0x2BFF";i:2;s:32:"Miscellaneous Symbols and Arrows";}i:78;a:3:{i:0;s:6:"0x2C00";i:1;s:6:"0x2C5F";i:2;s:10:"Glagolitic";}i:79;a:3:{i:0;s:6:"0x2C80";i:1;s:6:"0x2CFF";i:2;s:6:"Coptic";}i:80;a:3:{i:0;s:6:"0x2D00";i:1;s:6:"0x2D2F";i:2;s:19:"Georgian Supplement";}i:81;a:3:{i:0;s:6:"0x2D30";i:1;s:6:"0x2D7F";i:2;s:8:"Tifinagh";}i:82;a:3:{i:0;s:6:"0x2D80";i:1;s:6:"0x2DDF";i:2;s:17:"Ethiopic Extended";}i:83;a:3:{i:0;s:6:"0x2E00";i:1;s:6:"0x2E7F";i:2;s:24:"Supplemental Punctuation";}i:84;a:3:{i:0;s:6:"0x2E80";i:1;s:6:"0x2EFF";i:2;s:23:"CJK Radicals Supplement";}i:85;a:3:{i:0;s:6:"0x2F00";i:1;s:6:"0x2FDF";i:2;s:15:"Kangxi Radicals";}i:86;a:3:{i:0;s:6:"0x2FF0";i:1;s:6:"0x2FFF";i:2;s:34:"Ideographic Description Characters";}i:87;a:3:{i:0;s:6:"0x3000";i:1;s:6:"0x303F";i:2;s:27:"CJK Symbols and Punctuation";}i:88;a:3:{i:0;s:6:"0x3040";i:1;s:6:"0x309F";i:2;s:8:"Hiragana";}i:89;a:3:{i:0;s:6:"0x30A0";i:1;s:6:"0x30FF";i:2;s:8:"Katakana";}i:90;a:3:{i:0;s:6:"0x3100";i:1;s:6:"0x312F";i:2;s:8:"Bopomofo";}i:91;a:3:{i:0;s:6:"0x3130";i:1;s:6:"0x318F";i:2;s:25:"Hangul Compatibility Jamo";}i:92;a:3:{i:0;s:6:"0x3190";i:1;s:6:"0x319F";i:2;s:6:"Kanbun";}i:93;a:3:{i:0;s:6:"0x31A0";i:1;s:6:"0x31BF";i:2;s:17:"Bopomofo Extended";}i:94;a:3:{i:0;s:6:"0x31C0";i:1;s:6:"0x31EF";i:2;s:11:"CJK Strokes";}i:95;a:3:{i:0;s:6:"0x31F0";i:1;s:6:"0x31FF";i:2;s:28:"Katakana Phonetic Extensions";}i:96;a:3:{i:0;s:6:"0x3200";i:1;s:6:"0x32FF";i:2;s:31:"Enclosed CJK Letters and Months";}i:97;a:3:{i:0;s:6:"0x3300";i:1;s:6:"0x33FF";i:2;s:17:"CJK Compatibility";}i:98;a:3:{i:0;s:6:"0x3400";i:1;s:6:"0x4DBF";i:2;s:34:"CJK Unified Ideographs Extension A";}i:99;a:3:{i:0;s:6:"0x4DC0";i:1;s:6:"0x4DFF";i:2;s:23:"Yijing Hexagram Symbols";}i:100;a:3:{i:0;s:6:"0x4E00";i:1;s:6:"0x9FFF";i:2;s:22:"CJK Unified Ideographs";}i:101;a:3:{i:0;s:6:"0xA000";i:1;s:6:"0xA48F";i:2;s:12:"Yi Syllables";}i:102;a:3:{i:0;s:6:"0xA490";i:1;s:6:"0xA4CF";i:2;s:11:"Yi Radicals";}i:103;a:3:{i:0;s:6:"0xA700";i:1;s:6:"0xA71F";i:2;s:21:"Modifier Tone Letters";}i:104;a:3:{i:0;s:6:"0xA800";i:1;s:6:"0xA82F";i:2;s:12:"Syloti Nagri";}i:105;a:3:{i:0;s:6:"0xAC00";i:1;s:6:"0xD7AF";i:2;s:16:"Hangul Syllables";}i:106;a:3:{i:0;s:6:"0xD800";i:1;s:6:"0xDB7F";i:2;s:15:"High Surrogates";}i:107;a:3:{i:0;s:6:"0xDB80";i:1;s:6:"0xDBFF";i:2;s:27:"High Private Use Surrogates";}i:108;a:3:{i:0;s:6:"0xDC00";i:1;s:6:"0xDFFF";i:2;s:14:"Low Surrogates";}i:109;a:3:{i:0;s:6:"0xE000";i:1;s:6:"0xF8FF";i:2;s:16:"Private Use Area";}i:110;a:3:{i:0;s:6:"0xF900";i:1;s:6:"0xFAFF";i:2;s:28:"CJK Compatibility Ideographs";}i:111;a:3:{i:0;s:6:"0xFB00";i:1;s:6:"0xFB4F";i:2;s:29:"Alphabetic Presentation Forms";}i:112;a:3:{i:0;s:6:"0xFB50";i:1;s:6:"0xFDFF";i:2;s:27:"Arabic Presentation Forms-A";}i:113;a:3:{i:0;s:6:"0xFE00";i:1;s:6:"0xFE0F";i:2;s:19:"Variation Selectors";}i:114;a:3:{i:0;s:6:"0xFE10";i:1;s:6:"0xFE1F";i:2;s:14:"Vertical Forms";}i:115;a:3:{i:0;s:6:"0xFE20";i:1;s:6:"0xFE2F";i:2;s:20:"Combining Half Marks";}i:116;a:3:{i:0;s:6:"0xFE30";i:1;s:6:"0xFE4F";i:2;s:23:"CJK Compatibility Forms";}i:117;a:3:{i:0;s:6:"0xFE50";i:1;s:6:"0xFE6F";i:2;s:19:"Small Form Variants";}i:118;a:3:{i:0;s:6:"0xFE70";i:1;s:6:"0xFEFF";i:2;s:27:"Arabic Presentation Forms-B";}i:119;a:3:{i:0;s:6:"0xFF00";i:1;s:6:"0xFFEF";i:2;s:29:"Halfwidth and Fullwidth Forms";}i:120;a:3:{i:0;s:6:"0xFFF0";i:1;s:6:"0xFFFF";i:2;s:8:"Specials";}i:121;a:3:{i:0;s:7:"0x10000";i:1;s:7:"0x1007F";i:2;s:18:"Linear B Syllabary";}i:122;a:3:{i:0;s:7:"0x10080";i:1;s:7:"0x100FF";i:2;s:18:"Linear B Ideograms";}i:123;a:3:{i:0;s:7:"0x10100";i:1;s:7:"0x1013F";i:2;s:14:"Aegean Numbers";}i:124;a:3:{i:0;s:7:"0x10140";i:1;s:7:"0x1018F";i:2;s:21:"Ancient Greek Numbers";}i:125;a:3:{i:0;s:7:"0x10300";i:1;s:7:"0x1032F";i:2;s:10:"Old Italic";}i:126;a:3:{i:0;s:7:"0x10330";i:1;s:7:"0x1034F";i:2;s:6:"Gothic";}i:127;a:3:{i:0;s:7:"0x10380";i:1;s:7:"0x1039F";i:2;s:8:"Ugaritic";}i:128;a:3:{i:0;s:7:"0x103A0";i:1;s:7:"0x103DF";i:2;s:11:"Old Persian";}i:129;a:3:{i:0;s:7:"0x10400";i:1;s:7:"0x1044F";i:2;s:7:"Deseret";}i:130;a:3:{i:0;s:7:"0x10450";i:1;s:7:"0x1047F";i:2;s:7:"Shavian";}i:131;a:3:{i:0;s:7:"0x10480";i:1;s:7:"0x104AF";i:2;s:7:"Osmanya";}i:132;a:3:{i:0;s:7:"0x10800";i:1;s:7:"0x1083F";i:2;s:17:"Cypriot Syllabary";}i:133;a:3:{i:0;s:7:"0x10A00";i:1;s:7:"0x10A5F";i:2;s:10:"Kharoshthi";}i:134;a:3:{i:0;s:7:"0x1D000";i:1;s:7:"0x1D0FF";i:2;s:25:"Byzantine Musical Symbols";}i:135;a:3:{i:0;s:7:"0x1D100";i:1;s:7:"0x1D1FF";i:2;s:15:"Musical Symbols";}i:136;a:3:{i:0;s:7:"0x1D200";i:1;s:7:"0x1D24F";i:2;s:30:"Ancient Greek Musical Notation";}i:137;a:3:{i:0;s:7:"0x1D300";i:1;s:7:"0x1D35F";i:2;s:21:"Tai Xuan Jing Symbols";}i:138;a:3:{i:0;s:7:"0x1D400";i:1;s:7:"0x1D7FF";i:2;s:33:"Mathematical Alphanumeric Symbols";}i:139;a:3:{i:0;s:7:"0x20000";i:1;s:7:"0x2A6DF";i:2;s:34:"CJK Unified Ideographs Extension B";}i:140;a:3:{i:0;s:7:"0x2F800";i:1;s:7:"0x2FA1F";i:2;s:39:"CJK Compatibility Ideographs Supplement";}i:141;a:3:{i:0;s:7:"0xE0000";i:1;s:7:"0xE007F";i:2;s:4:"Tags";}i:142;a:3:{i:0;s:7:"0xE0100";i:1;s:7:"0xE01EF";i:2;s:30:"Variation Selectors Supplement";}i:143;a:3:{i:0;s:7:"0xF0000";i:1;s:7:"0xFFFFF";i:2;s:32:"Supplementary Private Use Area-A";}i:144;a:3:{i:0;s:8:"0x100000";i:1;s:8:"0x10FFFF";i:2;s:32:"Supplementary Private Use Area-B";}}
\ No newline at end of file
+a:145:{i:0;a:3:{i:0;i:0;i:1;i:127;i:2;s:11:"Basic Latin";}i:1;a:3:{i:0;i:128;i:1;i:255;i:2;s:18:"Latin-1 Supplement";}i:2;a:3:{i:0;i:256;i:1;i:383;i:2;s:16:"Latin Extended-A";}i:3;a:3:{i:0;i:384;i:1;i:591;i:2;s:16:"Latin Extended-B";}i:4;a:3:{i:0;i:592;i:1;i:687;i:2;s:14:"IPA Extensions";}i:5;a:3:{i:0;i:688;i:1;i:767;i:2;s:24:"Spacing Modifier Letters";}i:6;a:3:{i:0;i:768;i:1;i:879;i:2;s:27:"Combining Diacritical Marks";}i:7;a:3:{i:0;i:880;i:1;i:1023;i:2;s:16:"Greek and Coptic";}i:8;a:3:{i:0;i:1024;i:1;i:1279;i:2;s:8:"Cyrillic";}i:9;a:3:{i:0;i:1280;i:1;i:1327;i:2;s:19:"Cyrillic Supplement";}i:10;a:3:{i:0;i:1328;i:1;i:1423;i:2;s:8:"Armenian";}i:11;a:3:{i:0;i:1424;i:1;i:1535;i:2;s:6:"Hebrew";}i:12;a:3:{i:0;i:1536;i:1;i:1791;i:2;s:6:"Arabic";}i:13;a:3:{i:0;i:1792;i:1;i:1871;i:2;s:6:"Syriac";}i:14;a:3:{i:0;i:1872;i:1;i:1919;i:2;s:17:"Arabic Supplement";}i:15;a:3:{i:0;i:1920;i:1;i:1983;i:2;s:6:"Thaana";}i:16;a:3:{i:0;i:2304;i:1;i:2431;i:2;s:10:"Devanagari";}i:17;a:3:{i:0;i:2432;i:1;i:2559;i:2;s:7:"Bengali";}i:18;a:3:{i:0;i:2560;i:1;i:2687;i:2;s:8:"Gurmukhi";}i:19;a:3:{i:0;i:2688;i:1;i:2815;i:2;s:8:"Gujarati";}i:20;a:3:{i:0;i:2816;i:1;i:2943;i:2;s:5:"Oriya";}i:21;a:3:{i:0;i:2944;i:1;i:3071;i:2;s:5:"Tamil";}i:22;a:3:{i:0;i:3072;i:1;i:3199;i:2;s:6:"Telugu";}i:23;a:3:{i:0;i:3200;i:1;i:3327;i:2;s:7:"Kannada";}i:24;a:3:{i:0;i:3328;i:1;i:3455;i:2;s:9:"Malayalam";}i:25;a:3:{i:0;i:3456;i:1;i:3583;i:2;s:7:"Sinhala";}i:26;a:3:{i:0;i:3584;i:1;i:3711;i:2;s:4:"Thai";}i:27;a:3:{i:0;i:3712;i:1;i:3839;i:2;s:3:"Lao";}i:28;a:3:{i:0;i:3840;i:1;i:4095;i:2;s:7:"Tibetan";}i:29;a:3:{i:0;i:4096;i:1;i:4255;i:2;s:7:"Myanmar";}i:30;a:3:{i:0;i:4256;i:1;i:4351;i:2;s:8:"Georgian";}i:31;a:3:{i:0;i:4352;i:1;i:4607;i:2;s:11:"Hangul Jamo";}i:32;a:3:{i:0;i:4608;i:1;i:4991;i:2;s:8:"Ethiopic";}i:33;a:3:{i:0;i:4992;i:1;i:5023;i:2;s:19:"Ethiopic Supplement";}i:34;a:3:{i:0;i:5024;i:1;i:5119;i:2;s:8:"Cherokee";}i:35;a:3:{i:0;i:5120;i:1;i:5759;i:2;s:37:"Unified Canadian Aboriginal Syllabics";}i:36;a:3:{i:0;i:5760;i:1;i:5791;i:2;s:5:"Ogham";}i:37;a:3:{i:0;i:5792;i:1;i:5887;i:2;s:5:"Runic";}i:38;a:3:{i:0;i:5888;i:1;i:5919;i:2;s:7:"Tagalog";}i:39;a:3:{i:0;i:5920;i:1;i:5951;i:2;s:7:"Hanunoo";}i:40;a:3:{i:0;i:5952;i:1;i:5983;i:2;s:5:"Buhid";}i:41;a:3:{i:0;i:5984;i:1;i:6015;i:2;s:8:"Tagbanwa";}i:42;a:3:{i:0;i:6016;i:1;i:6143;i:2;s:5:"Khmer";}i:43;a:3:{i:0;i:6144;i:1;i:6319;i:2;s:9:"Mongolian";}i:44;a:3:{i:0;i:6400;i:1;i:6479;i:2;s:5:"Limbu";}i:45;a:3:{i:0;i:6480;i:1;i:6527;i:2;s:6:"Tai Le";}i:46;a:3:{i:0;i:6528;i:1;i:6623;i:2;s:11:"New Tai Lue";}i:47;a:3:{i:0;i:6624;i:1;i:6655;i:2;s:13:"Khmer Symbols";}i:48;a:3:{i:0;i:6656;i:1;i:6687;i:2;s:8:"Buginese";}i:49;a:3:{i:0;i:7424;i:1;i:7551;i:2;s:19:"Phonetic Extensions";}i:50;a:3:{i:0;i:7552;i:1;i:7615;i:2;s:30:"Phonetic Extensions Supplement";}i:51;a:3:{i:0;i:7616;i:1;i:7679;i:2;s:38:"Combining Diacritical Marks Supplement";}i:52;a:3:{i:0;i:7680;i:1;i:7935;i:2;s:25:"Latin Extended Additional";}i:53;a:3:{i:0;i:7936;i:1;i:8191;i:2;s:14:"Greek Extended";}i:54;a:3:{i:0;i:8192;i:1;i:8303;i:2;s:19:"General Punctuation";}i:55;a:3:{i:0;i:8304;i:1;i:8351;i:2;s:27:"Superscripts and Subscripts";}i:56;a:3:{i:0;i:8352;i:1;i:8399;i:2;s:16:"Currency Symbols";}i:57;a:3:{i:0;i:8400;i:1;i:8447;i:2;s:39:"Combining Diacritical Marks for Symbols";}i:58;a:3:{i:0;i:8448;i:1;i:8527;i:2;s:18:"Letterlike Symbols";}i:59;a:3:{i:0;i:8528;i:1;i:8591;i:2;s:12:"Number Forms";}i:60;a:3:{i:0;i:8592;i:1;i:8703;i:2;s:6:"Arrows";}i:61;a:3:{i:0;i:8704;i:1;i:8959;i:2;s:22:"Mathematical Operators";}i:62;a:3:{i:0;i:8960;i:1;i:9215;i:2;s:23:"Miscellaneous Technical";}i:63;a:3:{i:0;i:9216;i:1;i:9279;i:2;s:16:"Control Pictures";}i:64;a:3:{i:0;i:9280;i:1;i:9311;i:2;s:29:"Optical Character Recognition";}i:65;a:3:{i:0;i:9312;i:1;i:9471;i:2;s:22:"Enclosed Alphanumerics";}i:66;a:3:{i:0;i:9472;i:1;i:9599;i:2;s:11:"Box Drawing";}i:67;a:3:{i:0;i:9600;i:1;i:9631;i:2;s:14:"Block Elements";}i:68;a:3:{i:0;i:9632;i:1;i:9727;i:2;s:16:"Geometric Shapes";}i:69;a:3:{i:0;i:9728;i:1;i:9983;i:2;s:21:"Miscellaneous Symbols";}i:70;a:3:{i:0;i:9984;i:1;i:10175;i:2;s:8:"Dingbats";}i:71;a:3:{i:0;i:10176;i:1;i:10223;i:2;s:36:"Miscellaneous Mathematical Symbols-A";}i:72;a:3:{i:0;i:10224;i:1;i:10239;i:2;s:21:"Supplemental Arrows-A";}i:73;a:3:{i:0;i:10240;i:1;i:10495;i:2;s:16:"Braille Patterns";}i:74;a:3:{i:0;i:10496;i:1;i:10623;i:2;s:21:"Supplemental Arrows-B";}i:75;a:3:{i:0;i:10624;i:1;i:10751;i:2;s:36:"Miscellaneous Mathematical Symbols-B";}i:76;a:3:{i:0;i:10752;i:1;i:11007;i:2;s:35:"Supplemental Mathematical Operators";}i:77;a:3:{i:0;i:11008;i:1;i:11263;i:2;s:32:"Miscellaneous Symbols and Arrows";}i:78;a:3:{i:0;i:11264;i:1;i:11359;i:2;s:10:"Glagolitic";}i:79;a:3:{i:0;i:11392;i:1;i:11519;i:2;s:6:"Coptic";}i:80;a:3:{i:0;i:11520;i:1;i:11567;i:2;s:19:"Georgian Supplement";}i:81;a:3:{i:0;i:11568;i:1;i:11647;i:2;s:8:"Tifinagh";}i:82;a:3:{i:0;i:11648;i:1;i:11743;i:2;s:17:"Ethiopic Extended";}i:83;a:3:{i:0;i:11776;i:1;i:11903;i:2;s:24:"Supplemental Punctuation";}i:84;a:3:{i:0;i:11904;i:1;i:12031;i:2;s:23:"CJK Radicals Supplement";}i:85;a:3:{i:0;i:12032;i:1;i:12255;i:2;s:15:"Kangxi Radicals";}i:86;a:3:{i:0;i:12272;i:1;i:12287;i:2;s:34:"Ideographic Description Characters";}i:87;a:3:{i:0;i:12288;i:1;i:12351;i:2;s:27:"CJK Symbols and Punctuation";}i:88;a:3:{i:0;i:12352;i:1;i:12447;i:2;s:8:"Hiragana";}i:89;a:3:{i:0;i:12448;i:1;i:12543;i:2;s:8:"Katakana";}i:90;a:3:{i:0;i:12544;i:1;i:12591;i:2;s:8:"Bopomofo";}i:91;a:3:{i:0;i:12592;i:1;i:12687;i:2;s:25:"Hangul Compatibility Jamo";}i:92;a:3:{i:0;i:12688;i:1;i:12703;i:2;s:6:"Kanbun";}i:93;a:3:{i:0;i:12704;i:1;i:12735;i:2;s:17:"Bopomofo Extended";}i:94;a:3:{i:0;i:12736;i:1;i:12783;i:2;s:11:"CJK Strokes";}i:95;a:3:{i:0;i:12784;i:1;i:12799;i:2;s:28:"Katakana Phonetic Extensions";}i:96;a:3:{i:0;i:12800;i:1;i:13055;i:2;s:31:"Enclosed CJK Letters and Months";}i:97;a:3:{i:0;i:13056;i:1;i:13311;i:2;s:17:"CJK Compatibility";}i:98;a:3:{i:0;i:13312;i:1;i:19903;i:2;s:34:"CJK Unified Ideographs Extension A";}i:99;a:3:{i:0;i:19904;i:1;i:19967;i:2;s:23:"Yijing Hexagram Symbols";}i:100;a:3:{i:0;i:19968;i:1;i:40959;i:2;s:22:"CJK Unified Ideographs";}i:101;a:3:{i:0;i:40960;i:1;i:42127;i:2;s:12:"Yi Syllables";}i:102;a:3:{i:0;i:42128;i:1;i:42191;i:2;s:11:"Yi Radicals";}i:103;a:3:{i:0;i:42752;i:1;i:42783;i:2;s:21:"Modifier Tone Letters";}i:104;a:3:{i:0;i:43008;i:1;i:43055;i:2;s:12:"Syloti Nagri";}i:105;a:3:{i:0;i:44032;i:1;i:55215;i:2;s:16:"Hangul Syllables";}i:106;a:3:{i:0;i:55296;i:1;i:56191;i:2;s:15:"High Surrogates";}i:107;a:3:{i:0;i:56192;i:1;i:56319;i:2;s:27:"High Private Use Surrogates";}i:108;a:3:{i:0;i:56320;i:1;i:57343;i:2;s:14:"Low Surrogates";}i:109;a:3:{i:0;i:57344;i:1;i:63743;i:2;s:16:"Private Use Area";}i:110;a:3:{i:0;i:63744;i:1;i:64255;i:2;s:28:"CJK Compatibility Ideographs";}i:111;a:3:{i:0;i:64256;i:1;i:64335;i:2;s:29:"Alphabetic Presentation Forms";}i:112;a:3:{i:0;i:64336;i:1;i:65023;i:2;s:27:"Arabic Presentation Forms-A";}i:113;a:3:{i:0;i:65024;i:1;i:65039;i:2;s:19:"Variation Selectors";}i:114;a:3:{i:0;i:65040;i:1;i:65055;i:2;s:14:"Vertical Forms";}i:115;a:3:{i:0;i:65056;i:1;i:65071;i:2;s:20:"Combining Half Marks";}i:116;a:3:{i:0;i:65072;i:1;i:65103;i:2;s:23:"CJK Compatibility Forms";}i:117;a:3:{i:0;i:65104;i:1;i:65135;i:2;s:19:"Small Form Variants";}i:118;a:3:{i:0;i:65136;i:1;i:65279;i:2;s:27:"Arabic Presentation Forms-B";}i:119;a:3:{i:0;i:65280;i:1;i:65519;i:2;s:29:"Halfwidth and Fullwidth Forms";}i:120;a:3:{i:0;i:65520;i:1;i:65535;i:2;s:8:"Specials";}i:121;a:3:{i:0;i:65536;i:1;i:65663;i:2;s:18:"Linear B Syllabary";}i:122;a:3:{i:0;i:65664;i:1;i:65791;i:2;s:18:"Linear B Ideograms";}i:123;a:3:{i:0;i:65792;i:1;i:65855;i:2;s:14:"Aegean Numbers";}i:124;a:3:{i:0;i:65856;i:1;i:65935;i:2;s:21:"Ancient Greek Numbers";}i:125;a:3:{i:0;i:66304;i:1;i:66351;i:2;s:10:"Old Italic";}i:126;a:3:{i:0;i:66352;i:1;i:66383;i:2;s:6:"Gothic";}i:127;a:3:{i:0;i:66432;i:1;i:66463;i:2;s:8:"Ugaritic";}i:128;a:3:{i:0;i:66464;i:1;i:66527;i:2;s:11:"Old Persian";}i:129;a:3:{i:0;i:66560;i:1;i:66639;i:2;s:7:"Deseret";}i:130;a:3:{i:0;i:66640;i:1;i:66687;i:2;s:7:"Shavian";}i:131;a:3:{i:0;i:66688;i:1;i:66735;i:2;s:7:"Osmanya";}i:132;a:3:{i:0;i:67584;i:1;i:67647;i:2;s:17:"Cypriot Syllabary";}i:133;a:3:{i:0;i:68096;i:1;i:68191;i:2;s:10:"Kharoshthi";}i:134;a:3:{i:0;i:118784;i:1;i:119039;i:2;s:25:"Byzantine Musical Symbols";}i:135;a:3:{i:0;i:119040;i:1;i:119295;i:2;s:15:"Musical Symbols";}i:136;a:3:{i:0;i:119296;i:1;i:119375;i:2;s:30:"Ancient Greek Musical Notation";}i:137;a:3:{i:0;i:119552;i:1;i:119647;i:2;s:21:"Tai Xuan Jing Symbols";}i:138;a:3:{i:0;i:119808;i:1;i:120831;i:2;s:33:"Mathematical Alphanumeric Symbols";}i:139;a:3:{i:0;i:131072;i:1;i:173791;i:2;s:34:"CJK Unified Ideographs Extension B";}i:140;a:3:{i:0;i:194560;i:1;i:195103;i:2;s:39:"CJK Compatibility Ideographs Supplement";}i:141;a:3:{i:0;i:917504;i:1;i:917631;i:2;s:4:"Tags";}i:142;a:3:{i:0;i:917760;i:1;i:917999;i:2;s:30:"Variation Selectors Supplement";}i:143;a:3:{i:0;i:983040;i:1;i:1048575;i:2;s:32:"Supplementary Private Use Area-A";}i:144;a:3:{i:0;i:1048576;i:1;i:1114111;i:2;s:32:"Supplementary Private Use Area-B";}}
\ No newline at end of file
--- /dev/null
+<?php
+return array (
+ 0 =>
+ array (
+ 0 => 0x0000,
+ 1 => 0x007F,
+ 2 => 'Basic Latin',
+ ),
+ 1 =>
+ array (
+ 0 => 0x0080,
+ 1 => 0x00FF,
+ 2 => 'Latin-1 Supplement',
+ ),
+ 2 =>
+ array (
+ 0 => 0x0100,
+ 1 => 0x017F,
+ 2 => 'Latin Extended-A',
+ ),
+ 3 =>
+ array (
+ 0 => 0x0180,
+ 1 => 0x024F,
+ 2 => 'Latin Extended-B',
+ ),
+ 4 =>
+ array (
+ 0 => 0x0250,
+ 1 => 0x02AF,
+ 2 => 'IPA Extensions',
+ ),
+ 5 =>
+ array (
+ 0 => 0x02B0,
+ 1 => 0x02FF,
+ 2 => 'Spacing Modifier Letters',
+ ),
+ 6 =>
+ array (
+ 0 => 0x0300,
+ 1 => 0x036F,
+ 2 => 'Combining Diacritical Marks',
+ ),
+ 7 =>
+ array (
+ 0 => 0x0370,
+ 1 => 0x03FF,
+ 2 => 'Greek and Coptic',
+ ),
+ 8 =>
+ array (
+ 0 => 0x0400,
+ 1 => 0x04FF,
+ 2 => 'Cyrillic',
+ ),
+ 9 =>
+ array (
+ 0 => 0x0500,
+ 1 => 0x052F,
+ 2 => 'Cyrillic Supplement',
+ ),
+ 10 =>
+ array (
+ 0 => 0x0530,
+ 1 => 0x058F,
+ 2 => 'Armenian',
+ ),
+ 11 =>
+ array (
+ 0 => 0x0590,
+ 1 => 0x05FF,
+ 2 => 'Hebrew',
+ ),
+ 12 =>
+ array (
+ 0 => 0x0600,
+ 1 => 0x06FF,
+ 2 => 'Arabic',
+ ),
+ 13 =>
+ array (
+ 0 => 0x0700,
+ 1 => 0x074F,
+ 2 => 'Syriac',
+ ),
+ 14 =>
+ array (
+ 0 => 0x0750,
+ 1 => 0x077F,
+ 2 => 'Arabic Supplement',
+ ),
+ 15 =>
+ array (
+ 0 => 0x0780,
+ 1 => 0x07BF,
+ 2 => 'Thaana',
+ ),
+ 16 =>
+ array (
+ 0 => 0x0900,
+ 1 => 0x097F,
+ 2 => 'Devanagari',
+ ),
+ 17 =>
+ array (
+ 0 => 0x0980,
+ 1 => 0x09FF,
+ 2 => 'Bengali',
+ ),
+ 18 =>
+ array (
+ 0 => 0x0A00,
+ 1 => 0x0A7F,
+ 2 => 'Gurmukhi',
+ ),
+ 19 =>
+ array (
+ 0 => 0x0A80,
+ 1 => 0x0AFF,
+ 2 => 'Gujarati',
+ ),
+ 20 =>
+ array (
+ 0 => 0x0B00,
+ 1 => 0x0B7F,
+ 2 => 'Oriya',
+ ),
+ 21 =>
+ array (
+ 0 => 0x0B80,
+ 1 => 0x0BFF,
+ 2 => 'Tamil',
+ ),
+ 22 =>
+ array (
+ 0 => 0x0C00,
+ 1 => 0x0C7F,
+ 2 => 'Telugu',
+ ),
+ 23 =>
+ array (
+ 0 => 0x0C80,
+ 1 => 0x0CFF,
+ 2 => 'Kannada',
+ ),
+ 24 =>
+ array (
+ 0 => 0x0D00,
+ 1 => 0x0D7F,
+ 2 => 'Malayalam',
+ ),
+ 25 =>
+ array (
+ 0 => 0x0D80,
+ 1 => 0x0DFF,
+ 2 => 'Sinhala',
+ ),
+ 26 =>
+ array (
+ 0 => 0x0E00,
+ 1 => 0x0E7F,
+ 2 => 'Thai',
+ ),
+ 27 =>
+ array (
+ 0 => 0x0E80,
+ 1 => 0x0EFF,
+ 2 => 'Lao',
+ ),
+ 28 =>
+ array (
+ 0 => 0x0F00,
+ 1 => 0x0FFF,
+ 2 => 'Tibetan',
+ ),
+ 29 =>
+ array (
+ 0 => 0x1000,
+ 1 => 0x109F,
+ 2 => 'Myanmar',
+ ),
+ 30 =>
+ array (
+ 0 => 0x10A0,
+ 1 => 0x10FF,
+ 2 => 'Georgian',
+ ),
+ 31 =>
+ array (
+ 0 => 0x1100,
+ 1 => 0x11FF,
+ 2 => 'Hangul Jamo',
+ ),
+ 32 =>
+ array (
+ 0 => 0x1200,
+ 1 => 0x137F,
+ 2 => 'Ethiopic',
+ ),
+ 33 =>
+ array (
+ 0 => 0x1380,
+ 1 => 0x139F,
+ 2 => 'Ethiopic Supplement',
+ ),
+ 34 =>
+ array (
+ 0 => 0x13A0,
+ 1 => 0x13FF,
+ 2 => 'Cherokee',
+ ),
+ 35 =>
+ array (
+ 0 => 0x1400,
+ 1 => 0x167F,
+ 2 => 'Unified Canadian Aboriginal Syllabics',
+ ),
+ 36 =>
+ array (
+ 0 => 0x1680,
+ 1 => 0x169F,
+ 2 => 'Ogham',
+ ),
+ 37 =>
+ array (
+ 0 => 0x16A0,
+ 1 => 0x16FF,
+ 2 => 'Runic',
+ ),
+ 38 =>
+ array (
+ 0 => 0x1700,
+ 1 => 0x171F,
+ 2 => 'Tagalog',
+ ),
+ 39 =>
+ array (
+ 0 => 0x1720,
+ 1 => 0x173F,
+ 2 => 'Hanunoo',
+ ),
+ 40 =>
+ array (
+ 0 => 0x1740,
+ 1 => 0x175F,
+ 2 => 'Buhid',
+ ),
+ 41 =>
+ array (
+ 0 => 0x1760,
+ 1 => 0x177F,
+ 2 => 'Tagbanwa',
+ ),
+ 42 =>
+ array (
+ 0 => 0x1780,
+ 1 => 0x17FF,
+ 2 => 'Khmer',
+ ),
+ 43 =>
+ array (
+ 0 => 0x1800,
+ 1 => 0x18AF,
+ 2 => 'Mongolian',
+ ),
+ 44 =>
+ array (
+ 0 => 0x1900,
+ 1 => 0x194F,
+ 2 => 'Limbu',
+ ),
+ 45 =>
+ array (
+ 0 => 0x1950,
+ 1 => 0x197F,
+ 2 => 'Tai Le',
+ ),
+ 46 =>
+ array (
+ 0 => 0x1980,
+ 1 => 0x19DF,
+ 2 => 'New Tai Lue',
+ ),
+ 47 =>
+ array (
+ 0 => 0x19E0,
+ 1 => 0x19FF,
+ 2 => 'Khmer Symbols',
+ ),
+ 48 =>
+ array (
+ 0 => 0x1A00,
+ 1 => 0x1A1F,
+ 2 => 'Buginese',
+ ),
+ 49 =>
+ array (
+ 0 => 0x1D00,
+ 1 => 0x1D7F,
+ 2 => 'Phonetic Extensions',
+ ),
+ 50 =>
+ array (
+ 0 => 0x1D80,
+ 1 => 0x1DBF,
+ 2 => 'Phonetic Extensions Supplement',
+ ),
+ 51 =>
+ array (
+ 0 => 0x1DC0,
+ 1 => 0x1DFF,
+ 2 => 'Combining Diacritical Marks Supplement',
+ ),
+ 52 =>
+ array (
+ 0 => 0x1E00,
+ 1 => 0x1EFF,
+ 2 => 'Latin Extended Additional',
+ ),
+ 53 =>
+ array (
+ 0 => 0x1F00,
+ 1 => 0x1FFF,
+ 2 => 'Greek Extended',
+ ),
+ 54 =>
+ array (
+ 0 => 0x2000,
+ 1 => 0x206F,
+ 2 => 'General Punctuation',
+ ),
+ 55 =>
+ array (
+ 0 => 0x2070,
+ 1 => 0x209F,
+ 2 => 'Superscripts and Subscripts',
+ ),
+ 56 =>
+ array (
+ 0 => 0x20A0,
+ 1 => 0x20CF,
+ 2 => 'Currency Symbols',
+ ),
+ 57 =>
+ array (
+ 0 => 0x20D0,
+ 1 => 0x20FF,
+ 2 => 'Combining Diacritical Marks for Symbols',
+ ),
+ 58 =>
+ array (
+ 0 => 0x2100,
+ 1 => 0x214F,
+ 2 => 'Letterlike Symbols',
+ ),
+ 59 =>
+ array (
+ 0 => 0x2150,
+ 1 => 0x218F,
+ 2 => 'Number Forms',
+ ),
+ 60 =>
+ array (
+ 0 => 0x2190,
+ 1 => 0x21FF,
+ 2 => 'Arrows',
+ ),
+ 61 =>
+ array (
+ 0 => 0x2200,
+ 1 => 0x22FF,
+ 2 => 'Mathematical Operators',
+ ),
+ 62 =>
+ array (
+ 0 => 0x2300,
+ 1 => 0x23FF,
+ 2 => 'Miscellaneous Technical',
+ ),
+ 63 =>
+ array (
+ 0 => 0x2400,
+ 1 => 0x243F,
+ 2 => 'Control Pictures',
+ ),
+ 64 =>
+ array (
+ 0 => 0x2440,
+ 1 => 0x245F,
+ 2 => 'Optical Character Recognition',
+ ),
+ 65 =>
+ array (
+ 0 => 0x2460,
+ 1 => 0x24FF,
+ 2 => 'Enclosed Alphanumerics',
+ ),
+ 66 =>
+ array (
+ 0 => 0x2500,
+ 1 => 0x257F,
+ 2 => 'Box Drawing',
+ ),
+ 67 =>
+ array (
+ 0 => 0x2580,
+ 1 => 0x259F,
+ 2 => 'Block Elements',
+ ),
+ 68 =>
+ array (
+ 0 => 0x25A0,
+ 1 => 0x25FF,
+ 2 => 'Geometric Shapes',
+ ),
+ 69 =>
+ array (
+ 0 => 0x2600,
+ 1 => 0x26FF,
+ 2 => 'Miscellaneous Symbols',
+ ),
+ 70 =>
+ array (
+ 0 => 0x2700,
+ 1 => 0x27BF,
+ 2 => 'Dingbats',
+ ),
+ 71 =>
+ array (
+ 0 => 0x27C0,
+ 1 => 0x27EF,
+ 2 => 'Miscellaneous Mathematical Symbols-A',
+ ),
+ 72 =>
+ array (
+ 0 => 0x27F0,
+ 1 => 0x27FF,
+ 2 => 'Supplemental Arrows-A',
+ ),
+ 73 =>
+ array (
+ 0 => 0x2800,
+ 1 => 0x28FF,
+ 2 => 'Braille Patterns',
+ ),
+ 74 =>
+ array (
+ 0 => 0x2900,
+ 1 => 0x297F,
+ 2 => 'Supplemental Arrows-B',
+ ),
+ 75 =>
+ array (
+ 0 => 0x2980,
+ 1 => 0x29FF,
+ 2 => 'Miscellaneous Mathematical Symbols-B',
+ ),
+ 76 =>
+ array (
+ 0 => 0x2A00,
+ 1 => 0x2AFF,
+ 2 => 'Supplemental Mathematical Operators',
+ ),
+ 77 =>
+ array (
+ 0 => 0x2B00,
+ 1 => 0x2BFF,
+ 2 => 'Miscellaneous Symbols and Arrows',
+ ),
+ 78 =>
+ array (
+ 0 => 0x2C00,
+ 1 => 0x2C5F,
+ 2 => 'Glagolitic',
+ ),
+ 79 =>
+ array (
+ 0 => 0x2C80,
+ 1 => 0x2CFF,
+ 2 => 'Coptic',
+ ),
+ 80 =>
+ array (
+ 0 => 0x2D00,
+ 1 => 0x2D2F,
+ 2 => 'Georgian Supplement',
+ ),
+ 81 =>
+ array (
+ 0 => 0x2D30,
+ 1 => 0x2D7F,
+ 2 => 'Tifinagh',
+ ),
+ 82 =>
+ array (
+ 0 => 0x2D80,
+ 1 => 0x2DDF,
+ 2 => 'Ethiopic Extended',
+ ),
+ 83 =>
+ array (
+ 0 => 0x2E00,
+ 1 => 0x2E7F,
+ 2 => 'Supplemental Punctuation',
+ ),
+ 84 =>
+ array (
+ 0 => 0x2E80,
+ 1 => 0x2EFF,
+ 2 => 'CJK Radicals Supplement',
+ ),
+ 85 =>
+ array (
+ 0 => 0x2F00,
+ 1 => 0x2FDF,
+ 2 => 'Kangxi Radicals',
+ ),
+ 86 =>
+ array (
+ 0 => 0x2FF0,
+ 1 => 0x2FFF,
+ 2 => 'Ideographic Description Characters',
+ ),
+ 87 =>
+ array (
+ 0 => 0x3000,
+ 1 => 0x303F,
+ 2 => 'CJK Symbols and Punctuation',
+ ),
+ 88 =>
+ array (
+ 0 => 0x3040,
+ 1 => 0x309F,
+ 2 => 'Hiragana',
+ ),
+ 89 =>
+ array (
+ 0 => 0x30A0,
+ 1 => 0x30FF,
+ 2 => 'Katakana',
+ ),
+ 90 =>
+ array (
+ 0 => 0x3100,
+ 1 => 0x312F,
+ 2 => 'Bopomofo',
+ ),
+ 91 =>
+ array (
+ 0 => 0x3130,
+ 1 => 0x318F,
+ 2 => 'Hangul Compatibility Jamo',
+ ),
+ 92 =>
+ array (
+ 0 => 0x3190,
+ 1 => 0x319F,
+ 2 => 'Kanbun',
+ ),
+ 93 =>
+ array (
+ 0 => 0x31A0,
+ 1 => 0x31BF,
+ 2 => 'Bopomofo Extended',
+ ),
+ 94 =>
+ array (
+ 0 => 0x31C0,
+ 1 => 0x31EF,
+ 2 => 'CJK Strokes',
+ ),
+ 95 =>
+ array (
+ 0 => 0x31F0,
+ 1 => 0x31FF,
+ 2 => 'Katakana Phonetic Extensions',
+ ),
+ 96 =>
+ array (
+ 0 => 0x3200,
+ 1 => 0x32FF,
+ 2 => 'Enclosed CJK Letters and Months',
+ ),
+ 97 =>
+ array (
+ 0 => 0x3300,
+ 1 => 0x33FF,
+ 2 => 'CJK Compatibility',
+ ),
+ 98 =>
+ array (
+ 0 => 0x3400,
+ 1 => 0x4DBF,
+ 2 => 'CJK Unified Ideographs Extension A',
+ ),
+ 99 =>
+ array (
+ 0 => 0x4DC0,
+ 1 => 0x4DFF,
+ 2 => 'Yijing Hexagram Symbols',
+ ),
+ 100 =>
+ array (
+ 0 => 0x4E00,
+ 1 => 0x9FFF,
+ 2 => 'CJK Unified Ideographs',
+ ),
+ 101 =>
+ array (
+ 0 => 0xA000,
+ 1 => 0xA48F,
+ 2 => 'Yi Syllables',
+ ),
+ 102 =>
+ array (
+ 0 => 0xA490,
+ 1 => 0xA4CF,
+ 2 => 'Yi Radicals',
+ ),
+ 103 =>
+ array (
+ 0 => 0xA700,
+ 1 => 0xA71F,
+ 2 => 'Modifier Tone Letters',
+ ),
+ 104 =>
+ array (
+ 0 => 0xA800,
+ 1 => 0xA82F,
+ 2 => 'Syloti Nagri',
+ ),
+ 105 =>
+ array (
+ 0 => 0xAC00,
+ 1 => 0xD7AF,
+ 2 => 'Hangul Syllables',
+ ),
+ 106 =>
+ array (
+ 0 => 0xD800,
+ 1 => 0xDB7F,
+ 2 => 'High Surrogates',
+ ),
+ 107 =>
+ array (
+ 0 => 0xDB80,
+ 1 => 0xDBFF,
+ 2 => 'High Private Use Surrogates',
+ ),
+ 108 =>
+ array (
+ 0 => 0xDC00,
+ 1 => 0xDFFF,
+ 2 => 'Low Surrogates',
+ ),
+ 109 =>
+ array (
+ 0 => 0xE000,
+ 1 => 0xF8FF,
+ 2 => 'Private Use Area',
+ ),
+ 110 =>
+ array (
+ 0 => 0xF900,
+ 1 => 0xFAFF,
+ 2 => 'CJK Compatibility Ideographs',
+ ),
+ 111 =>
+ array (
+ 0 => 0xFB00,
+ 1 => 0xFB4F,
+ 2 => 'Alphabetic Presentation Forms',
+ ),
+ 112 =>
+ array (
+ 0 => 0xFB50,
+ 1 => 0xFDFF,
+ 2 => 'Arabic Presentation Forms-A',
+ ),
+ 113 =>
+ array (
+ 0 => 0xFE00,
+ 1 => 0xFE0F,
+ 2 => 'Variation Selectors',
+ ),
+ 114 =>
+ array (
+ 0 => 0xFE10,
+ 1 => 0xFE1F,
+ 2 => 'Vertical Forms',
+ ),
+ 115 =>
+ array (
+ 0 => 0xFE20,
+ 1 => 0xFE2F,
+ 2 => 'Combining Half Marks',
+ ),
+ 116 =>
+ array (
+ 0 => 0xFE30,
+ 1 => 0xFE4F,
+ 2 => 'CJK Compatibility Forms',
+ ),
+ 117 =>
+ array (
+ 0 => 0xFE50,
+ 1 => 0xFE6F,
+ 2 => 'Small Form Variants',
+ ),
+ 118 =>
+ array (
+ 0 => 0xFE70,
+ 1 => 0xFEFF,
+ 2 => 'Arabic Presentation Forms-B',
+ ),
+ 119 =>
+ array (
+ 0 => 0xFF00,
+ 1 => 0xFFEF,
+ 2 => 'Halfwidth and Fullwidth Forms',
+ ),
+ 120 =>
+ array (
+ 0 => 0xFFF0,
+ 1 => 0xFFFF,
+ 2 => 'Specials',
+ ),
+ 121 =>
+ array (
+ 0 => 0x10000,
+ 1 => 0x1007F,
+ 2 => 'Linear B Syllabary',
+ ),
+ 122 =>
+ array (
+ 0 => 0x10080,
+ 1 => 0x100FF,
+ 2 => 'Linear B Ideograms',
+ ),
+ 123 =>
+ array (
+ 0 => 0x10100,
+ 1 => 0x1013F,
+ 2 => 'Aegean Numbers',
+ ),
+ 124 =>
+ array (
+ 0 => 0x10140,
+ 1 => 0x1018F,
+ 2 => 'Ancient Greek Numbers',
+ ),
+ 125 =>
+ array (
+ 0 => 0x10300,
+ 1 => 0x1032F,
+ 2 => 'Old Italic',
+ ),
+ 126 =>
+ array (
+ 0 => 0x10330,
+ 1 => 0x1034F,
+ 2 => 'Gothic',
+ ),
+ 127 =>
+ array (
+ 0 => 0x10380,
+ 1 => 0x1039F,
+ 2 => 'Ugaritic',
+ ),
+ 128 =>
+ array (
+ 0 => 0x103A0,
+ 1 => 0x103DF,
+ 2 => 'Old Persian',
+ ),
+ 129 =>
+ array (
+ 0 => 0x10400,
+ 1 => 0x1044F,
+ 2 => 'Deseret',
+ ),
+ 130 =>
+ array (
+ 0 => 0x10450,
+ 1 => 0x1047F,
+ 2 => 'Shavian',
+ ),
+ 131 =>
+ array (
+ 0 => 0x10480,
+ 1 => 0x104AF,
+ 2 => 'Osmanya',
+ ),
+ 132 =>
+ array (
+ 0 => 0x10800,
+ 1 => 0x1083F,
+ 2 => 'Cypriot Syllabary',
+ ),
+ 133 =>
+ array (
+ 0 => 0x10A00,
+ 1 => 0x10A5F,
+ 2 => 'Kharoshthi',
+ ),
+ 134 =>
+ array (
+ 0 => 0x1D000,
+ 1 => 0x1D0FF,
+ 2 => 'Byzantine Musical Symbols',
+ ),
+ 135 =>
+ array (
+ 0 => 0x1D100,
+ 1 => 0x1D1FF,
+ 2 => 'Musical Symbols',
+ ),
+ 136 =>
+ array (
+ 0 => 0x1D200,
+ 1 => 0x1D24F,
+ 2 => 'Ancient Greek Musical Notation',
+ ),
+ 137 =>
+ array (
+ 0 => 0x1D300,
+ 1 => 0x1D35F,
+ 2 => 'Tai Xuan Jing Symbols',
+ ),
+ 138 =>
+ array (
+ 0 => 0x1D400,
+ 1 => 0x1D7FF,
+ 2 => 'Mathematical Alphanumeric Symbols',
+ ),
+ 139 =>
+ array (
+ 0 => 0x20000,
+ 1 => 0x2A6DF,
+ 2 => 'CJK Unified Ideographs Extension B',
+ ),
+ 140 =>
+ array (
+ 0 => 0x2F800,
+ 1 => 0x2FA1F,
+ 2 => 'CJK Compatibility Ideographs Supplement',
+ ),
+ 141 =>
+ array (
+ 0 => 0xE0000,
+ 1 => 0xE007F,
+ 2 => 'Tags',
+ ),
+ 142 =>
+ array (
+ 0 => 0xE0100,
+ 1 => 0xE01EF,
+ 2 => 'Variation Selectors Supplement',
+ ),
+ 143 =>
+ array (
+ 0 => 0xF0000,
+ 1 => 0xFFFFF,
+ 2 => 'Supplementary Private Use Area-A',
+ ),
+ 144 =>
+ array (
+ 0 => 0x100000,
+ 1 => 0x10FFFF,
+ 2 => 'Supplementary Private Use Area-B',
+ ),
+);
git.wh0rd.org / tt-rss.git / commitdiff
