FM / tokenizer

/opt/canhelp/node_modules/sucrase/dist/esm/parser/tokenizer/

/opt/canhelp/node_modules/sucrase/dist/esm/parser/tokenizer

Name	Size	Mode	Actions
index.js	29255	0644	edit dl rm
keywords.js	4276	0644	edit dl rm
readWord.js	1940	0644	edit dl rm
readWordTree.js	43212	0644	edit dl rm
state.js	3845	0644	edit dl rm
types.js	17970	0644	edit dl rm

Edit: /opt/canhelp/node_modules/sucrase/dist/esm/parser/tokenizer/readWord.js (1940B)

import {input, state} from "../traverser/base";
import {charCodes} from "../util/charcodes";
import {IS_IDENTIFIER_CHAR} from "../util/identifier";
import {finishToken} from "./index";
import {READ_WORD_TREE} from "./readWordTree";
import {TokenType as tt} from "./types";

/**
 * Read an identifier, producing either a name token or matching on one of the existing keywords.
 * For performance, we pre-generate big decision tree that we traverse. Each node represents a
 * prefix and has 27 values, where the first value is the token or contextual token, if any (-1 if
 * not), and the other 26 values are the transitions to other nodes, or -1 to stop.
 */
export default function readWord() {
  let treePos = 0;
  let code = 0;
  let pos = state.pos;
  while (pos < input.length) {
    code = input.charCodeAt(pos);
    if (code < charCodes.lowercaseA || code > charCodes.lowercaseZ) {
      break;
    }
    const next = READ_WORD_TREE[treePos + (code - charCodes.lowercaseA) + 1];
    if (next === -1) {
      break;
    } else {
      treePos = next;
      pos++;
    }
  }

while (pos < input.length) {
    const ch = input.charCodeAt(pos);
    if (IS_IDENTIFIER_CHAR[ch]) {
      pos++;
    } else if (ch === charCodes.backslash) {
      // \u
      pos += 2;
      if (input.charCodeAt(pos) === charCodes.leftCurlyBrace) {
        while (pos < input.length && input.charCodeAt(pos) !== charCodes.rightCurlyBrace) {
          pos++;
        }
        pos++;
      }
    } else if (ch === charCodes.atSign && input.charCodeAt(pos + 1) === charCodes.atSign) {
      pos += 2;
    } else {
      break;
    }
  }
  state.pos = pos;
  finishToken(tt.name);
}

cmd: