lexgo

package module

v0.0.0-...-76311a6 Latest Latest Go to latest Published: Sep 8, 2025 License: MIT Imports: 8 Imported by: 3

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

codeberg.org/pierrec/lexgo

Links

Open Source Insights

README ¶

Package lexgo provides a simple lexer.

Usage

package parser

import "codeberg.org/pierrec/lexgo"

type Token uint8

const (
	Invalid Token = iota
	Error
	EOF
	Unsigned
	Signed
	Float
	String
	Identifier

	Add
	Sub
	Mul
	Div
)

func init() {
	config := lexgo.Config[Token]{
		Invalid:    Invalid,
		Error:      Error,
		EOF:        EOF,
		Unsigned:   Unsigned,
		Signed:     Signed,
		Float:      Float,
		String:     String,
		Identifier: Identifier,
		All: map[Token]string{
			Add: "+",
			Sub: "-",
			Mul: "*",
			Div: "/",
		},
	}

	r := strings.NewReader("1 + 2")
	lex := lexgo.New[Token](r, config)
	switch lex.Next() {
	case Unsigned:
	case Add:
	}
}

Documentation ¶

Overview ¶

Package lexgo implements a simple lexer.

Index ¶

Variables
type Config
type Lexer
- func New[T comparable](r io.Reader, config Config[T]) (*Lexer[T], error)
type Location
- func (loc Location) String() string
type ValueFormat
- func (f ValueFormat) Format(v uint64) string

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	ErrInvalidHex         = errors.New("invalid hex")
	ErrUnterminatedString = errors.New("unterminated string")
	ErrUnterminatedChar   = errors.New("unterminated char")
)

Functions ¶

This section is empty.

Types ¶

type Config ¶

type Config[T comparable] struct {
	// IsIdentChar determines if a given byte is part of an identifier based on its position within the string.
	// The default function matches digits (except as first char), lower and upper letters, underscore.
	IsIdentChar func(c byte, index int) bool
	// EscapeChar is used to unescape a character literal.
	EscapeChar func(escaped byte, r io.ByteReader) (unescaped byte, _ error)
	Invalid    T
	Error      T
	EOF        T
	Unsigned   T
	Signed     T
	Float      T
	String     T
	Identifier T
	All        map[T]string
}

type Lexer ¶

type Lexer[T comparable] struct {
	Error       error
	Location    Location
	String      string // identifier, comment, string literal
	ValueFormat ValueFormat
	Unsigned    uint64
	Signed      int64
	Float       float64
	// contains filtered or unexported fields
}

func New ¶

func New[T comparable](r io.Reader, config Config[T]) (*Lexer[T], error)

func (*Lexer[T]) HasWhitespace ¶

func (lex *Lexer[T]) HasWhitespace() bool

HasWhitespace returns whether the next token is a whitespace without consuming it.

func (*Lexer[T]) Init ¶

func (lex *Lexer[T]) Init(r io.Reader, config Config[T]) error

func (*Lexer[T]) Newline ¶

func (lex *Lexer[T]) Newline() bool

Newline returns whether a newline was encountered in the last Peek() or Next() call. NB. whitespace is always skipped at the start when calling Peek() and Next().

func (*Lexer[T]) Next ¶

func (lex *Lexer[T]) Next() T

func (*Lexer[T]) NextLine ¶

func (lex *Lexer[T]) NextLine() string

func (*Lexer[T]) Peek ¶

func (lex *Lexer[T]) Peek() T

func (*Lexer[T]) ReadByte ¶

func (lex *Lexer[T]) ReadByte() (byte, error)

func (*Lexer[T]) Reset ¶

func (lex *Lexer[T]) Reset(src io.Reader)

type Location ¶

type Location struct {
	File   string
	Offset uint32
	Line   uint32
	Column uint32
}

func (Location) String ¶

func (loc Location) String() string

type ValueFormat ¶

type ValueFormat uint8

const (
	Decimal     ValueFormat = iota // 123
	Hexadecimal                    // 0x
	Binary                         // 0b
	Character                      // ' '
)

func (ValueFormat) Format ¶

func (f ValueFormat) Format(v uint64) string

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL