#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Transliterate texts between unicode and standard transliteration schemes. Transliterate texts between non-latin scripts and commonly-used latin transliteration schemes. Uses standard Unicode character blocks -- e.g. DEVANAGARI U+0900 ... U+097F -- and transliteration schemes -- e.g. the IAST convention for transliteration of Sanskrit to latin-with-dots. The following character blocks and transliteration schemes are included: DEVANAGARI IAST ITRANS -- http://www.aczoom.com/itrans/#itransencoding (Sanskrit only) Harvard Kyoto CYRILLIC ISO 9:1995 (Russian only) New character blocks and transliteration schemes can be added by creating new CharacterBlock and TransliterationScheme objects. COMMAND LINE USAGE ---------------------------- python transliterator.py text inputFormat outputFormat ... writes the transliterated text to stdout text -- the text to be transliterated OR the name of a file containing the text inputFormat -- the name of the character block or transliteration scheme that the text is to be transliterated FROM, e.g. 'CYRILLIC', 'IAST'. Not case-sensitive outputFormat -- the name of the character block or transliteration scheme that the text is to be transliterated TO, e.g. 'CYRILLIC', 'IAST'. Not case-sensitive USAGE -------- Transliterate a text: >>> import indic_transliteration_unmaintained.little.transliterator >>> transliterator.transliterate('yogazcittavRttinirodhaH', 'harvardkyoto', ... 'devanagari', {'outputASCIIEncoded' : True}) 'योगश्चित्तवृत्तिनिरोधः' Create a new CharacterBlock and TransliterationScheme: >>> import indic_transliteration_unmaintained.little.transliterator >>> cb = transliterator.CharacterBlock('NEWBLOCK', range(0x901, 0x9FF)) >>> scheme = transliterator.TransliterationScheme(cb.name, 'NEWSCHEME', ... {'ab': 0x901, 'cd': 0x902}) >>> transliterator.transliterate('abcd', scheme, cb, {'outputASCIIEncoded' : True}) 'ँं' COPYRIGHT AND DISCLAIMER ------------------------------------ Transliterator is: version 0.1 software - use at your own risk. The IAST, ITRANS and Harvard-Kyoto transliteration schemes have been tested for classical Sanskrit, not for any other language. The Cyrillic alphabet and ISO 9:1995 transliteration (for Russian only) are included but have been even more lightly tested than Devanagari. Copyright (c) 2005 by Alan Little By obtaining, using, and/or copying this software and/or its associated documentation, you agree that you have read, understood, and will comply with the following terms and conditions: Permission to use, copy, modify, and distribute this software and its associated documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appears in all copies, and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of the author not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ """ TO DO cyrillic: GOST & something ASCII-only & sensible punctuation &/ numerals: HK check Itrans punctuation implicit conversion to unicode if no to format specified URLs for the input text Bugs is there a problem with implicit A before visarga? """ __version__ = '0.1' import unicodedata import doctest import sys characterBlocks = {} _names = {} # handle unrecognised characters UNRECOGNISED_FAIL = 0 UNRECOGNISED_ECHO = 1 UNRECOGNISED_SUBSTITUTE = 2 # default options options = {} def resetOptions(): """ Reset options to their default values. """ global options defaultOptions = { 'inputEncoding' : 'utf-8', # default input encoding for strings 'outputEncoding' : 'utf-8', # default output encoding 'substituteChar' : '?', # use to substitute unrecognised characters 'handleUnrecognised' : UNRECOGNISED_ECHO, # unrecognised characters: # fail, echo or substitute 'outputASCIIEncoded' : False, # HTML-encoded ASCII output? } options = defaultOptions.copy() resetOptions() def _unrecognised(chr): """ Handle unrecognised characters. """ if options['handleUnrecognised'] == UNRECOGNISED_ECHO: return chr elif options['handleUnrecognised'] == UNRECOGNISED_SUBSTITUTE: return options['substituteChar'] else: raise (KeyError, chr) class TLCharacter (object): """ Class representing a Unicode character with its equivalents. Public attributes: unicodeHexValue -- the numeric value of the Unicode code point. chr -- the character value of the Unicode code point. name -- the name of the Unicode code point. equivalents -- a dict containing the character's equivalents in various transliteration schemes, in the format: {'Scheme A': 'A', 'Scheme B': 'aah', } where keys are TransliterationScheme names, values are transliterated equivalents of the character. """ def __init__(self, unicodeHexValue, block): """ Set up a unicode character. Arguments: unicodeHexValue -- an integer that should correspond to a Unicode code point. block -- the CharacterBlock this character belongs to. Raises: ValueError -- if unicodeHexValue is not a valid code point. """ if unicodeHexValue < 0 or unicodeHexValue > 0x10FFFF: raise (ValueError, "numeric value outside Unicode range") self.unicodeHexValue = unicodeHexValue """ Use name check to filter out unused characters. unicodedata.name() raises ValueError for these """ self.chr = chr(self.unicodeHexValue) self.name = unicodedata.name(self.chr) self.equivalents = {} self._block = block def addEquivalent(self, equivName, equivalent): """ Add an equivalent for the character. Arguments: equivName -- the name of a TransliterationScheme equivalent -- string/unicode equivalent in the named TransliterationScheme for this code point. """ self.equivalents[equivName] = equivalent class CharacterBlock(dict): """ Dictionary-like representation of a set of unicode characters. For our purposes, a character block corresponds to an alphabet/script that we want to be able to transliterate to or from, e.g. Cyrillic, Devanagari. Keys are unicode characters. Values are TLCharacter instances. """ def __init__(self, name, charRange, charClass=TLCharacter): """ Set up a character block corresponding to a range of code points. Keyword arguments: name -- a string containing the name of the character block. (should normally use a standard Unicode character block name) range -- a list of code points. Reserved code points are ignored. charClass -- the class to be used to create the characters. Should be a subclass of TLCharacter. """ """ Ensure that any character sequence dependencies will be ok. e.g. set up Devanagari standalone vowels before dependents. """ charRange.sort() for c in charRange: try: tlchar = charClass(c, self) self[tlchar.chr] = tlchar except ValueError: # Unicode reserved code points. # not an error pass self._longestEntry = 1 self.name = name self.transliterationSchemes = {} self._register() def _register(self): characterBlocks[self.name] = self _names[self.name.upper()] = self def _transliterate (self, text, outFormat): """ Transliterate the text to the target transliteration scheme.""" result = [] for c in text: if c.isspace(): result.append(c) try: result.append(self[c].equivalents[outFormat.name]) except KeyError: result.append(_unrecognised(c)) return result def _preprocess(self, text): """ Make our signature compatible with TransliterationScheme. """ return text def _getNextChar(self, text, startPos): return text[startPos] class TransliterationScheme(dict): """ Dictionary-like representation of a transliteration scheme. e.g. the Harvard-Kyoto, IAST or ITRANS schemes for transliterating Devanagari to or from the latin alphabet. Keys are unicode strings representing the letter-equivalents used in the transliteration scheme. Values are TLCharacter instances. """ def __init__(self, blockName, schemeName, data, swapTable=None): """ Set up a transliteration scheme. Keyword arguments: blockName -- a string containg the name of the character block this transliteration scheme is used for, e.g. 'CYRILLIC', 'DEVANAGARI'. schemeName -- the name of the transliteration scheme. Must be unique. data -- a dict containing the data for the transliteration scheme. Keys are transliterated Unicode characters or strings. Values are integers corresponding to Unicode code points. For examples, see the data for the built-in transliteration schemes. swapTable -- a dict (default None) containing any non-standard letter combinations used in the transliteration scheme that we want to pre-process away before transliterating. See the ITRANS data for examples. Raises: KeyError: unknown block name. TypeError: swapTable is not a dict """ self.block = characterBlocks[blockName] self.name = schemeName for equiv, unicodeHexValue in data.items(): self[equiv] = self.block[chr(unicodeHexValue)] self[equiv].addEquivalent(self.name, equiv) self._longestEntry = max([len(e) for e in data.keys()]) if self._longestEntry > 1: self._parseTree = {} self._parsedata = list(data.keys()) self._parsedata.sort() self._setupParseTree(0, len(data) - 1, 0, self._parseTree) if swapTable is not None: if not isinstance(swapTable, dict): raise (TypeError) self.swapTable = swapTable self._register() def _register(self): self.block.transliterationSchemes[self.name] = self _names[self.name.upper()] = self def _setupParseTree(self, rowFrom, rowTo, colIndex, tree): """ Build the search tree for multi-character encodings. """ if colIndex == self._longestEntry: return prevchar = None rowIndex = rowFrom while rowIndex <= rowTo: if colIndex < len(self._parsedata[rowIndex]): c = self._parsedata[rowIndex][colIndex] if c != prevchar: tree[c] = {} if prevchar is not None: self._setupParseTree(rowFrom, rowIndex - 1, colIndex + 1, tree[prevchar]) rowFrom = rowIndex prevchar = c if rowIndex == rowTo: self._setupParseTree(rowFrom, rowIndex, colIndex + 1, tree[prevchar]) rowIndex = rowIndex + 1 def __getNextChar(self, text, startPos, tree): i = startPos matched = text[i] if i < len(text) - 1: try: if text[i + 1] in tree[text[i]]: matched = matched + self.__getNextChar(text, i + 1, tree[text[i]]) except KeyError: # Ignore. The lookup for the equivalent character later on will handle it. pass return matched def _getNextChar(self, text, startPos): if self._longestEntry > 1 and not text[startPos].isspace(): return self.__getNextChar(text, startPos, self._parseTree) else: return text[startPos] def _preprocess(self, text): if self.swapTable: for c in self.swapTable: text = text.replace(c, self.swapTable[c]) return text def _transliterate (self, text, outFormat): """ Transliterate the text to Unicode.""" result = [] text = self._preprocess(text) i = 0 while i < len(text): if text[i].isspace(): result.append(text[i]) i = i+1 else: chr = self._getNextChar(text, i) try: result.append(self[chr].chr) except KeyError: result.append(_unrecognised(chr)) i = i + len(chr) return result def transliterate(text, inFormat, outFormat, requestOptions={}): """ Transliterate a text. Keyword arguments: text -- a unicode string containing the text to be transliterated inFormat -- the "from" CharacterBlock or TransliterationScheme, or its name outFormat -- the target CharacterBlock or TransliterationScheme, or its name requestOptions -- optional dict containing option settings that override the defaults for this request. Returns a unicode object containing the text transliterated into the target character set. Raises: ValueError -- unrecognised input or output format. KeyError -- a character in text is not a member of inFormat, or has no corresponding character defined in outFormat. """ def asciiEncode(chr): value = ord(chr) if value > 255: return '&#x%x;' % (value) return chr try: options.update(requestOptions) """ Ensure we have the correct encoding for input text. """ # if isinstance(text, str): # text = text.decode(options['inputEncoding']) basestring = (str, bytes) """ Look up input & output format names. """ def findFormat(fmt): if isinstance(fmt, basestring): try: fmt = _names[fmt.upper()] except KeyError: raise (ValueError, 'unrecognised format ' + fmt) return fmt inFormat = findFormat(inFormat) outFormat = findFormat(outFormat) """ Perform sanity checks. """ if not isinstance(text, basestring): raise (TypeError, "The text must be a string or a unicode object") def getBlock(format): if isinstance(format, CharacterBlock): return format else: return format.block inBlock = getBlock(inFormat) outBlock = getBlock(outFormat) if not inBlock is outBlock: raise (ValueError, "incompatible input and output formats") if inFormat is outFormat: # They're trying to trick us. Just do a quick sanity check & bounce it back. if inFormat._longestEntry == 1: [inFormat[c] for c in set(text) if not c.isspace()] # -> KeyError for extraneous chars. return text """ At last we're happy. Do it. """ result = inFormat._transliterate(text, outFormat) if options['outputASCIIEncoded']: result = [asciiEncode(c) for c in result] return u''.join(result).encode(options['outputEncoding']) finally: resetOptions() """ DEVANAGARI PROCESSING Specialised classes & functions to handle Devanagari. """ class DevanagariCharacter (TLCharacter): """ Special processing for Devanagari characters. """ """ Devanaagarii characters need to know if they are vowels or not. Unicode Data doesn't help with this - category 'Mn' is not unique to dependent vowels - so need to hard code the ranges """ _vowelOffset = 0x93E - 0x906 _depVowelRange = range(0x93E, 0x94D) _vowelRange = range(0x904, 0x915) _VIRAMA = '्' _LETTER_A = "अ" """ Unicode calls agravaha a letter. Not for our purposes: we need to not treat it as one for handling virama & implicit 'a' """ _AVAGRAHA = 0x93D _OM = 0x950 def __init__(self, unicodeHexValue, block): """ Create an object representing a Devanagari character. Extends TLCharacter.__init__ to distinguish Devanagari standalone vowels, dependent vowels and consonants. Raises ValueError -- for characters in the Devanagari dependent vowel range. We want these as variants of the corresponding standalone vowels, not as separate characters. """ self.isVowel = False if unicodeHexValue in DevanagariCharacter._vowelRange: self.isVowel = True self._dependentVowel = None if unicodeHexValue in DevanagariCharacter._depVowelRange: # We are a dependent vowel. # Find the corresponding normal vowel. vowel = block[chr(unicodeHexValue - DevanagariCharacter._vowelOffset)] vowel._setDependentVowel(unicodeHexValue) raise (ValueError) # don't create dependent vowels as separate instances TLCharacter.__init__(self, unicodeHexValue, block) self.isConsonant = False if self.isVowel == False \ and self.chr.isalpha() \ and self.unicodeHexValue not in (DevanagariCharacter._AVAGRAHA, DevanagariCharacter._OM): self.isConsonant = True def _setDependentVowel(self, unicodeHexValue): if unicodeHexValue is not None: if not self.isVowel: raise (ValueError) self._dependentVowel = chr(unicodeHexValue) self._block[chr(unicodeHexValue)] = self class _Devanagari(object): """ Holder class for the Devanagari transliteration algorithm. """ def _transliterate(self, text, outFormat): """ Transliterate a devanagari text into the target format. Transliterating a character to or from Devanagari is not a simple lookup: it depends on the preceding and following characters. """ def getResult(): if curMatch.isspace(): result.append(curMatch) return if prevMatch in self: prev = self[prevMatch] else: prev = None if nextMatch in self: next = self[nextMatch] else: next = None try: equiv = outFormat._equivalent(self[curMatch], prev, #self.get(prevMatch, None), next, #self.get(nextMatch, None), self._implicitA) except KeyError: equiv = _unrecognised(curMatch) for e in equiv: result.append(e) def incr(c): if self._longestEntry == 1: return 1 return len(c) result = [] text = self._preprocess(text) i = 0 prevMatch = None nextMatch = None curMatch = self._getNextChar(text, i) i = i + len(curMatch) while i < len(text): nextMatch = self._getNextChar(text, i) getResult() i = i + len(nextMatch) prevMatch = curMatch curMatch = nextMatch nextMatch = None getResult() return result class DevanagariCharacterBlock(CharacterBlock, _Devanagari): """ Class representing the Devanagari Unicode character block. """ def __init__(self, name, charRange): """ Set up the Devanagari character block. Extends CharacterBlock.__init__ by specifiying that the characters created should be instances of DevanagariCharacter. """ CharacterBlock.__init__(self, name, charRange, DevanagariCharacter) self._implicitA = True # generate implicit As when transliterating # *FROM* this scheme def _transliterate(self, text, outFormat): """ Need to specify which superclass _transliterate() to call. """ return _Devanagari._transliterate(self, text, outFormat) def _equivalent(self, char, prev, next, implicitA): """ Transliterate a Latin character equivalent to Devanagari. Add VIRAMA for ligatures. Convert standalone to dependent vowels. """ result = [] if char.isVowel == False: result.append(char.chr) if char.isConsonant \ and ((next is not None and next.isConsonant) \ or next is None): result.append(DevanagariCharacter._VIRAMA) else: if prev is None or prev.isConsonant == False: result.append(char.chr) else: if char._dependentVowel is not None: result.append(char._dependentVowel) return result class DevanagariTransliterationScheme(TransliterationScheme, _Devanagari): """ Class representing a Devanagari transliteration scheme. """ def __init__(self, blockName, schemeName, data, swapTable=None): """ Set up a Devanagari transliteration scheme. Extends TransliterationScheme.__init__ """ TransliterationScheme.__init__\ (self, blockName, schemeName, data, swapTable) self._implicitA = False # generate implicit As when transliterating # *FROM* this scheme def _transliterate(self, text, outFormat): """ Need to specify which superclass _transliterate() to call. """ return _Devanagari._transliterate(self, text, outFormat) def _equivalent(self, char, prev, next, implicitA): """ Transliterate a Devanagari character to Latin. Add implicit As unless overridden by VIRAMA. """ implicitA = False # Force it! result = [] if char.chr != DevanagariCharacter._VIRAMA: result.append(char.equivalents[self.name]) """ Append implicit A to consonants if the next character isn't a vowel. """ if implicitA and char.isConsonant \ and ((next is not None \ and next.chr != DevanagariCharacter._VIRAMA \ and not next.isVowel) \ or next is None): result.append(characterBlocks['DEVANAGARI']\ [DevanagariCharacter._LETTER_A].equivalents[self.name]) return result """ DEVANAGARI DATA set up the Devanagari character set with three commonly used transliteration schemes: ITRANS Harvard Kyoto IAST """ DevanagariCharacterBlock('DEVANAGARI', list(range(0x900, 0x97F))) HARVARDKYOTO = { \ 'M': 0x902, 'H': 0x903, 'a': 0x905, 'A': 0x906, 'i': 0x907, 'I': 0x908, 'u': 0x909, 'U': 0x90A, 'R': 0x90B, 'lR': 0x90C, 'e': 0x90F, 'ai': 0x910, 'o': 0x913, 'au': 0x914, 'k': 0x915, 'kh': 0x916, 'g': 0x917, 'gh': 0x918, 'G': 0x919, 'c': 0x91A, 'ch': 0x91B, 'j': 0x91C, 'jh': 0x91D, 'J': 0x91E, 'T': 0x91F, 'Th': 0x920, 'D': 0x921, 'Dh': 0x922, 'N': 0x923, 't': 0x924, 'th': 0x925, 'd': 0x926, 'dh': 0x927, 'n': 0x928, 'p': 0x92A, 'ph': 0x92B, 'b': 0x92C, 'bh': 0x92D, 'm': 0x92E, 'y': 0x92F, 'r': 0x930, 'l': 0x932, 'L': 0x933, 'v': 0x935, 'z': 0x936, 'zh': 0x934, # added for Tamil 'S': 0x937, 's': 0x938, 'h': 0x939, '.': 0x0964, '..': 0x0965, "'": 0x93D, 'oM': 0x950, } DevanagariTransliterationScheme('DEVANAGARI','HARVARDKYOTO', HARVARDKYOTO) ITRANS = { \ 'M': 0x902, '.n': 0x902, '.m': 0x902, 'H': 0x903, 'a': 0x905, 'A': 0x906, 'aa': 0x906, 'i': 0x907, 'I': 0x908, 'ii': 0x908, 'u': 0x909, 'U': 0x90A, 'uu': 0x90A, 'RRi': 0x90B, 'R^i': 0x90B, 'LLi': 0x90C, 'L^i': 0x90C, 'e': 0x90F, 'ai': 0x910, 'o': 0x913, 'au': 0x914, 'k': 0x915, 'kh': 0x916, 'g': 0x917, 'gh': 0x918, '~N': 0x919, 'c': 0x91A, 'C': 0x91B, 'ch': 0x91A, 'Ch': 0x91B, 'j': 0x91C, 'jh': 0x91D, '~n': 0x91E, 'T': 0x91F, 'Th': 0x920, 'D': 0x921, 'Dh': 0x922, 'N': 0x923, 't': 0x924, 'th': 0x925, 'd': 0x926, 'dh': 0x927, 'n': 0x928, 'p': 0x92A, 'ph': 0x92B, 'b': 0x92C, 'bh': 0x92D, 'm': 0x92E, 'y': 0x92F, 'r': 0x930, 'l': 0x932, 'L': 0x933, 'v': 0x935, 'w': 0x935, 'sh': 0x936, 'Sh': 0x937, 's': 0x938, 'h': 0x939, ".a": 0x93D, # avagraha 'OM': 0x950, 'AUM': 0x950, '.': 0x0964, '..': 0x0965, '0': 0x0966, '1': 0x0967, '2': 0x0968, '3': 0x0969, '4': 0x096A, '5': 0x096B, '6': 0x096C, '7': 0x096D, '8': 0x096E, '9': 0x096F, # non-standard/obsolete iTrans variants still used in texts from # http://sanskrit.gde.to/ '.h': 0x903, 'N^': 0x919, 'shh': 0x937, 'JN': 0x91E, } """ ITrans uses some characters only in common ligatures. The easiest way to deal with these is to replace them with their "normal consonant" equivalents before we try to transliterate. (This assumes we are mainly transliterating iTrans inbound, and that the normal consonants are acceptable outbound. ITrans is not a good choice for outbound anyway because it has so many ambiguities) """ _swapTable = {'GY': 'j~n', 'dny': 'j~n', 'x': 'kSh', } DevanagariTransliterationScheme('DEVANAGARI', 'ITRANS', ITRANS, _swapTable) IAST = { \ chr(0x1E43): 0x902, chr(0x1E25): 0x903, 'a': 0x905, chr(0x101): 0x906, 'i': 0x907, chr(0x12B): 0x908, 'u': 0x909, chr(0x16B): 0x90A, chr(0x1E5B): 0x90B, chr(0x1E37): 0x90C, 'e': 0x90F, 'ai': 0x910, 'o': 0x913, 'au': 0x914, 'k': 0x915, 'kh': 0x916, 'g': 0x917, 'gh': 0x918, chr(0x1E45): 0x919, 'ch': 0x91A, # deliberate change, for ease of reading 'chh': 0x91B, # deliberate change, for ease of reading 'j': 0x91C, 'jh': 0x91D, chr(0xF1): 0x91E, chr(0x1E6D): 0x91F, chr(0x1E6D) +'h': 0x920, chr(0x1E0D): 0x921, chr(0x1E0D) + 'h': 0x922, chr(0x1E47): 0x923, 't': 0x924, 'th': 0x925, 'd': 0x926, 'dh': 0x927, 'n': 0x928, 'p': 0x92A, 'ph': 0x92B, 'b': 0x92C, 'bh': 0x92D, 'm': 0x92E, 'y': 0x92F, 'r': 0x930, 'l': 0x932, 'L': 0x933, 'zh': 0x934, # Added for Tamil 'v': 0x935, chr(0x15B): 0x936, chr(0x1E63): 0x937, 's': 0x938, 'h': 0x939, "'": 0x93D, # avagraha 'O' + chr(0x1E43): 0x950, '.': 0x0964, '..': 0x0965, '0': 0x0966, '1': 0x0967, '2': 0x0968, '3': 0x0969, '4': 0x096A, '5': 0x096B, '6': 0x096C, '7': 0x096D, '8': 0x096E, '9': 0x096F, } DevanagariTransliterationScheme('DEVANAGARI', 'IAST', IAST) """ CYRILLIC DATA set up Cyrillic and the ISO 9:1995 (Russian) transliteration scheme. The Cyrillic unicode range contains about 4x what contemporary Russian uses - other languages & history wacky stuff. Set the full range up in case anybody ever has occasion to use it. """ CharacterBlock('CYRILLIC', list(range(0x400, 0x510))) _ISO9RUS = {\ chr(0x0CB): 0x401, # IO 'A': 0x410, 'B': 0x411, 'V': 0x412, 'G': 0x413, 'D': 0x414, 'E': 0x415, chr(0x17D): 0x416, # ZHE 'Z': 0x417, 'I': 0x418, 'J': 0x419, 'K': 0x41a, 'L': 0x41b, 'M': 0x41c, 'N': 0x41d, 'O': 0x41e, 'P': 0x41f, 'R': 0x420, 'S': 0x421, 'T': 0x422, 'U': 0x423, 'F': 0x424, 'H': 0x425, 'C': 0x426, # TS chr(0x10C): 0x427, # CH chr(0x160): 0x428, # SH chr(0x15C): 0x429, # SHCH chr(0x2BA): 0x42a, # hard 'Y': 0x42b, chr(0x2B9): 0x42c, # soft chr(0x0C8): 0x42d, # YE chr(0x0DB): 0x42e, # YU chr(0x0C2): 0x42f, # YA 'a': 0x430, 'b': 0x431, 'v': 0x432, 'g': 0x433, 'd': 0x434, 'e': 0x435, chr(0x17E): 0x436, # zhe 'z': 0x437, 'i': 0x438, 'j': 0x439, 'k': 0x43a, 'l': 0x43b, 'm': 0x43c, 'n': 0x43d, 'o': 0x43e, 'p': 0x43f, 'r': 0x440, 's': 0x441, 't': 0x442, 'u': 0x443, 'f': 0x444, 'h': 0x445, 'c': 0x446, # ts chr(0x10D): 0x447, # ch chr(0x161): 0x448, # sh chr(0x15D): 0x449, # shch # chr(0x2BA): 0x44a, # hard - same upper & lowercase 'y': 0x44b, # chr(0x2B9): 0x44c, # soft - same upper & lowercase chr(0xE8): 0x44d, # ye chr(0x0FB): 0x44e, #yu chr(0x0E2): 0x44f, # ya chr(0x0EB): 0x451, # io } TransliterationScheme('CYRILLIC', 'ISO9RUS', _ISO9RUS) CharacterBlock('TAMIL', list(range(0x0b82, 0x0bd0))) HKTAM = { \ 'M': 0x0b82, 'H': 0x0b83, 'a': 0x0b85, 'A': 0x0b86, 'i': 0x0b87, 'I': 0x0b88, 'u': 0x0b89, 'U': 0x0b8a, # 'R': 0x90B, # 'lR': 0x90C, 'e': 0x0b8e, 'E': 0x0b8f, 'ai': 0x0b90, 'o': 0x0b92, 'O': 0x0b93, 'au': 0x0b94, 'k': 0x0b95, 'kh': 0x0b95, 'g': 0x0b95, 'gh': 0x0b95, 'G': 0x0b99, 'c': 0x0b9a, 'ch': 0x0b9a, 'j': 0x0b9c, 'jh': 0x0b9c, 'J': 0x0b9e, 'T': 0x0b9f, 'Th': 0x0b9f, 'D': 0x0b9f, 'Dh': 0x0b9f, 'N': 0x0ba3, 't': 0x0ba4, 'th': 0x0ba4, 'd': 0x0ba4, 'dh': 0x0ba4, 'n': 0x0ba8, 'p': 0x0baa, 'ph': 0x0baa, 'b': 0x0baa, 'bh': 0x0baa, 'm': 0x0bae, 'y': 0x0baf, 'r': 0x0bb0, 'l': 0x0bb2, 'L': 0x0bb3, 'v': 0x0bb5, 'z': 0x0bb6, 'zh': 0x0bb4, # added for Tamil 'S': 0x0bb7, 's': 0x0bb8, 'h': 0x0bb9, # '.': 0x0964, # '..': 0x0965, # "'": 0x93D, 'oM': 0x0bd0, } # TransliterationScheme('TAMIL','HKTAM', HKTAM) def main(argv=None): """ Call transliterator from a command line. python transliterator.py text inputFormat outputFormat ... writes the transliterated text to stdout text -- the text to be transliterated OR the name of a file containing the text inputFormat -- the name of the character block or transliteration scheme that the text is to be transliterated FROM, e.g. 'CYRILLIC', 'IAST'. Not case-sensitive outputFormat -- the name of the character block or transliteration scheme that the text is to be transliterated TO, e.g. 'CYRILLIC', 'IAST'. Not case-sensitive """ print (transliterate('jaya gaNeza! zrIrAmajayam', 'harvardkyoto', 'devanagari')) if argv is None: argv = sys.argv try: text, inFormat, outFormat = argv[1:4] except ValueError: print (main.__doc__) return 2 inFormat = inFormat.upper() outFormat = outFormat.upper() # try assuming "text" is a filename try: f = open(text) except IOError: # it wasn't, so it must be the actual text print (transliterate(text, inFormat, outFormat)) return 0 else: i = 1 for text in f.readlines(): if len(text) > 0 and not text.startswith('#'): print (transliterate(text, inFormat, outFormat).strip('\n')) i = i + 1 f.close() return 0 if __name__ == "__main__": sys.exit(main())