FM / chardet

/usr/lib/python3/dist-packages/chardet

Name	Size	Mode	Actions
cli/	-	0755	rm
metadata/	-	0755	rm
__pycache__/	-	0755	rm
big5freq.py	31254	0644	edit dl rm
big5prober.py	1757	0644	edit dl rm
chardistribution.py	9411	0644	edit dl rm
charsetgroupprober.py	3839	0644	edit dl rm
charsetprober.py	5110	0644	edit dl rm
codingstatemachine.py	3590	0644	edit dl rm
compat.py	1200	0644	edit dl rm
cp949prober.py	1855	0644	edit dl rm
enums.py	1661	0644	edit dl rm
escprober.py	3950	0644	edit dl rm
escsm.py	10510	0644	edit dl rm
eucjpprober.py	3749	0644	edit dl rm
euckrfreq.py	13546	0644	edit dl rm
euckrprober.py	1748	0644	edit dl rm
euctwfreq.py	31621	0644	edit dl rm
euctwprober.py	1747	0644	edit dl rm
gb2312freq.py	20715	0644	edit dl rm
gb2312prober.py	1754	0644	edit dl rm
hebrewprober.py	13838	0644	edit dl rm
jisfreq.py	25777	0644	edit dl rm
jpcntx.py	19643	0644	edit dl rm
langbulgarianmodel.py	105685	0644	edit dl rm
langgreekmodel.py	99559	0644	edit dl rm
langhebrewmodel.py	98764	0644	edit dl rm
langhungarianmodel.py	102486	0644	edit dl rm
langrussianmodel.py	131168	0644	edit dl rm
langthaimodel.py	103300	0644	edit dl rm
langturkishmodel.py	95934	0644	edit dl rm
latin1prober.py	5370	0644	edit dl rm
mbcharsetprober.py	3413	0644	edit dl rm
mbcsgroupprober.py	2012	0644	edit dl rm
mbcssm.py	25481	0644	edit dl rm
sbcharsetprober.py	6136	0644	edit dl rm
sbcsgroupprober.py	4309	0644	edit dl rm
sjisprober.py	3774	0644	edit dl rm
universaldetector.py	12503	0644	edit dl rm
utf8prober.py	2766	0644	edit dl rm
version.py	242	0644	edit dl rm
__init__.py	3271	0644	edit dl rm

Edit: /usr/lib/python3/dist-packages/chardet/charsetprober.py (5110B)

######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

import logging
import re

from .enums import ProbingState

class CharSetProber(object):

SHORTCUT_THRESHOLD = 0.95

def __init__(self, lang_filter=None):
        self._state = None
        self.lang_filter = lang_filter
        self.logger = logging.getLogger(__name__)

def reset(self):
        self._state = ProbingState.DETECTING

@property
    def charset_name(self):
        return None

def feed(self, buf):
        pass

@property
    def state(self):
        return self._state

def get_confidence(self):
        return 0.0

@staticmethod
    def filter_high_byte_only(buf):
        buf = re.sub(b'([\x00-\x7F])+', b' ', buf)
        return buf

@staticmethod
    def filter_international_words(buf):
        """
        We define three types of bytes:
        alphabet: english alphabets [a-zA-Z]
        international: international characters [\x80-\xFF]
        marker: everything else [^a-zA-Z\x80-\xFF]

The input buffer can be thought to contain a series of words delimited
        by markers. This function works to filter all words that contain at
        least one international character. All contiguous sequences of markers
        are replaced by a single space ascii character.

This filter applies to all scripts which do not use English characters.
        """
        filtered = bytearray()

# This regex expression filters out only words that have at-least one
        # international character. The word may include one marker character at
        # the end.
        words = re.findall(b'[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?',
                           buf)

for word in words:
            filtered.extend(word[:-1])

# If the last character in the word is a marker, replace it with a
            # space as markers shouldn't affect our analysis (they are used
            # similarly across all languages and may thus have similar
            # frequencies).
            last_char = word[-1:]
            if not last_char.isalpha() and last_char < b'\x80':
                last_char = b' '
            filtered.extend(last_char)

return filtered

@staticmethod
    def filter_with_english_letters(buf):
        """
        Returns a copy of ``buf`` that retains only the sequences of English
        alphabet and high byte characters that are not between <> characters.
        Also retains English alphabet and high byte characters immediately
        before occurrences of >.

This filter can be applied to all scripts which contain both English
        characters and extended ASCII characters, but is currently only used by
        ``Latin1Prober``.
        """
        filtered = bytearray()
        in_tag = False
        prev = 0

for curr in range(len(buf)):
            # Slice here to get bytes instead of an int with Python 3
            buf_char = buf[curr:curr + 1]
            # Check if we're coming out of or entering an HTML tag
            if buf_char == b'>':
                in_tag = False
            elif buf_char == b'<':
                in_tag = True

# If current character is not extended-ASCII and not alphabetic...
            if buf_char < b'\x80' and not buf_char.isalpha():
                # ...and we're not in a tag
                if curr > prev and not in_tag:
                    # Keep everything after last non-extended-ASCII,
                    # non-alphabetic character
                    filtered.extend(buf[prev:curr])
                    # Output a space to delimit stretch we kept
                    filtered.extend(b' ')
                prev = curr + 1

# If we're not in a tag...
        if not in_tag:
            # Keep everything after last non-extended-ASCII, non-alphabetic
            # character
            filtered.extend(buf[prev:])

return filtered

cmd: