Gecko [ gandhischool.edu.np ]

Name	Size	Permission
Enum.js	651 B	-rw-rw-rw-
Middleware.js	3.27 KB	-rw-rw-rw-
Parser.js	5.83 KB	-rw-rw-rw-
Prefixer.js	6.37 KB	-rw-rw-rw-
Serializer.js	1.09 KB	-rw-rw-rw-
Tokenizer.js	4.84 KB	-rw-rw-rw-
Utility.js	2.25 KB	-rw-rw-rw-

Code Editor : Tokenizer.js

import {from, trim, charat, strlen, substr, append, assign} from './Utility.js'

export var line = 1
export var column = 1
export var length = 0
export var position = 0
export var character = 0
export var characters = ''

/**
 * @param {string} value
 * @param {object | null} root
 * @param {object | null} parent
 * @param {string} type
 * @param {string[] | string} props
 * @param {object[] | string} children
 * @param {object[]} siblings
 * @param {number} length
 */
export function node (value, root, parent, type, props, children, length, siblings) {
	return {value: value, root: root, parent: parent, type: type, props: props, children: children, line: line, column: column, length: length, return: '', siblings: siblings}
}

/**
 * @param {object} root
 * @param {object} props
 * @return {object}
 */
export function copy (root, props) {
	return assign(node('', null, null, '', null, null, 0, root.siblings), root, {length: -root.length}, props)
}

/**
 * @param {object} root
 */
export function lift (root) {
	while (root.root)
		root = copy(root.root, {children: [root]})

append(root, root.siblings)
}

/**
 * @return {number}
 */
export function char () {
	return character
}

/**
 * @return {number}
 */
export function prev () {
	character = position > 0 ? charat(characters, --position) : 0

if (column--, character === 10)
		column = 1, line--

return character
}

/**
 * @return {number}
 */
export function next () {
	character = position < length ? charat(characters, position++) : 0

if (column++, character === 10)
		column = 1, line++

return character
}

/**
 * @return {number}
 */
export function peek () {
	return charat(characters, position)
}

/**
 * @return {number}
 */
export function caret () {
	return position
}

/**
 * @param {number} begin
 * @param {number} end
 * @return {string}
 */
export function slice (begin, end) {
	return substr(characters, begin, end)
}

/**
 * @param {number} type
 * @return {number}
 */
export function token (type) {
	switch (type) {
		// \0 \t \n \r \s whitespace token
		case 0: case 9: case 10: case 13: case 32:
			return 5
		// ! + , / > @ ~ isolate token
		case 33: case 43: case 44: case 47: case 62: case 64: case 126:
		// ; { } breakpoint token
		case 59: case 123: case 125:
			return 4
		// : accompanied token
		case 58:
			return 3
		// " ' ( [ opening delimit token
		case 34: case 39: case 40: case 91:
			return 2
		// ) ] closing delimit token
		case 41: case 93:
			return 1
	}

return 0
}

/**
 * @param {string} value
 * @return {any[]}
 */
export function alloc (value) {
	return line = column = 1, length = strlen(characters = value), position = 0, []
}

/**
 * @param {any} value
 * @return {any}
 */
export function dealloc (value) {
	return characters = '', value
}

/**
 * @param {number} type
 * @return {string}
 */
export function delimit (type) {
	return trim(slice(position - 1, delimiter(type === 91 ? type + 2 : type === 40 ? type + 1 : type)))
}

/**
 * @param {string} value
 * @return {string[]}
 */
export function tokenize (value) {
	return dealloc(tokenizer(alloc(value)))
}

/**
 * @param {number} type
 * @return {string}
 */
export function whitespace (type) {
	while (character = peek())
		if (character < 33)
			next()
		else
			break

return token(type) > 2 || token(character) > 3 ? '' : ' '
}

/**
 * @param {string[]} children
 * @return {string[]}
 */
export function tokenizer (children) {
	while (next())
		switch (token(character)) {
			case 0: append(identifier(position - 1), children)
				break
			case 2: append(delimit(character), children)
				break
			default: append(from(character), children)
		}

return children
}

/**
 * @param {number} index
 * @param {number} count
 * @return {string}
 */
export function escaping (index, count) {
	while (--count && next())
		// not 0-9 A-F a-f
		if (character < 48 || character > 102 || (character > 57 && character < 65) || (character > 70 && character < 97))
			break

return slice(index, caret() + (count < 6 && peek() == 32 && next() == 32))
}

/**
 * @param {number} type
 * @return {number}
 */
export function delimiter (type) {
	while (next())
		switch (character) {
			// ] ) " '
			case type:
				return position
			// " '
			case 34: case 39:
				if (type !== 34 && type !== 39)
					delimiter(character)
				break
			// (
			case 40:
				if (type === 41)
					delimiter(type)
				break
			// \
			case 92:
				next()
				break
		}

return position
}

/**
 * @param {number} type
 * @param {number} index
 * @return {number}
 */
export function commenter (type, index) {
	while (next())
		// //
		if (type + character === 47 + 10)
			break
		// /*
		else if (type + character === 42 + 42 && peek() === 47)
			break

return '/*' + slice(index, position - 1) + '*' + from(type === 47 ? type : next())
}

/**
 * @param {number} index
 * @return {string}
 */
export function identifier (index) {
	while (!token(peek()))
		next()

return slice(index, position)
}

${this.title}

Code Editor : Tokenizer.js