Source code for nltk.corpus.reader.udhr

# -*- coding: utf-8 -*-
"""
UDHR corpus reader. It mostly deals with encodings.
"""
from __future__ import absolute_import, unicode_literals

from nltk.corpus.reader.util import find_corpus_fileids
from nltk.corpus.reader.plaintext import PlaintextCorpusReader

class UdhrCorpusReader(PlaintextCorpusReader):

    ENCODINGS = [
        ('.*-Latin1$', 'latin-1'),
        ('.*-Hebrew$', 'hebrew'),
        ('.*-Arabic$', 'cp1256'),
        ('Czech_Cesky-UTF8', 'cp1250'), # yeah
        ('.*-Cyrillic$', 'cyrillic'),
        ('.*-SJIS$', 'SJIS'),
        ('.*-GB2312$', 'GB2312'),
        ('.*-Latin2$', 'ISO-8859-2'),
        ('.*-Greek$', 'greek'),
        ('.*-UTF8$', 'utf-8'),

        ('Hungarian_Magyar-Unicode', 'utf-16-le'),
        ('Amahuaca', 'latin1'),
        ('Turkish_Turkce-Turkish', 'latin5'),
        ('Lithuanian_Lietuviskai-Baltic', 'latin4'),
        ('Japanese_Nihongo-EUC', 'EUC-JP'),
        ('Japanese_Nihongo-JIS', 'iso2022_jp'),
        ('Chinese_Mandarin-HZ', 'hz'),
        ('Abkhaz\-Cyrillic\+Abkh', 'cp1251'),
    ]

    SKIP = set([
        # The following files are not fully decodable because they
        # were truncated at wrong bytes:
        'Burmese_Myanmar-UTF8',
        'Japanese_Nihongo-JIS',
        'Chinese_Mandarin-HZ',
        'Chinese_Mandarin-UTF8',
        'Gujarati-UTF8',
        'Hungarian_Magyar-Unicode',
        'Lao-UTF8',
        'Magahi-UTF8',
        'Marathi-UTF8',
        'Tamil-UTF8',

        # Unfortunately, encodings required for reading
        # the following files are not supported by Python:
        'Vietnamese-VPS',
        'Vietnamese-VIQR',
        'Vietnamese-TCVN',
        'Magahi-Agra',
        'Bhojpuri-Agra',
        'Esperanto-T61', # latin3 raises an exception

        # The following files are encoded for specific fonts:
        'Burmese_Myanmar-WinResearcher',
        'Armenian-DallakHelv',
        'Tigrinya_Tigrigna-VG2Main',
        'Amharic-Afenegus6..60375', # ?
        'Navaho_Dine-Navajo-Navaho-font',

        # What are these?
        'Azeri_Azerbaijani_Cyrillic-Az.Times.Cyr.Normal0117',
        'Azeri_Azerbaijani_Latin-Az.Times.Lat0117',

        # The following files are unintended:
        'Czech-Latin2-err',
        'Russian_Russky-UTF8~',
    ])


    def __init__(self, root='udhr'):
        fileids = find_corpus_fileids(root, r'(?!README|\.).*')
        super(UdhrCorpusReader, self).__init__(
            root,
            [fileid for fileid in fileids if fileid not in self.SKIP],
            encoding=self.ENCODINGS
        )