From 732c4e668485eec0cb82d82962ba55ae21984eca Mon Sep 17 00:00:00 2001 From: metif12 Date: Sun, 4 Oct 2026 17:03:16 +0330 Subject: [PATCH] csplit: implement it csplit splits a file into pieces at the lines its patterns name, and reports the size of each piece in bytes. All five pattern forms and all seven options are implemented: an integer, /REGEXP/[OFFSET], %REGEXP%[OFFSET], {INTEGER} and {*}; and -b, -f, -k, --suppress-matched, -n, -s and -z. The pattern language is a GNU basic regular expression, not the POSIX extended one the manual's word "REGEXP" suggests. Measured against csplit 9.4 with one-line inputs, where a match shows up as an empty first output file: ^a+$ matches "a+" but not "aa" so + is an ordinary character ^a\?$ matches "a" so \? is the optional operator ^ab{1}$ matches "ab{1}" so { is an ordinary character ^a|b$ matches "a|b" so | is an ordinary character ^\(a\)$ matches "a" so \( and \) are the grouping forms ^a^b$ matches "a^b" so ^ anchors only at the very start ^\101$ is refused as a back reference V's own regex module is not a substitute for that: its | binds to single tokens instead of to sequences, and it has neither POSIX character classes nor \b, so even a two-branch alternation cannot be expressed there. ere.v and ere_parser.v are therefore a small matcher for the dialect above, carrying the measurements in their headers so the rules can be checked rather than re-derived. The tests assert the measured values rather than comparing against the platform utility, and they drive the binary through os.new_process. That is not a style choice: os.execute is deprecated and fails under -W, and it cannot hand a child any stdin, which is the only way an operand of "-" can be reached from a test. Verified on Windows against GNU csplit 9.4 under WSL over 277 cases comparing the file names, the file contents, stdout, stderr and the exit code: 275 match. The two that do not are a piece name containing "|", which Windows will not create, and --suppress-matched=1, which the flag module accepts where GNU refuses an argument to a boolean. Both are noted rather than worked around. --- src/csplit/args.v | 246 ++++++++++++++++++ src/csplit/csplit.v | 513 ++++++++++++++++++++++++++++++++++++++ src/csplit/csplit_test.v | 507 +++++++++++++++++++++++++++++++++++++ src/csplit/ere.v | 277 +++++++++++++++++++++ src/csplit/ere_parser.v | 523 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 2066 insertions(+) create mode 100644 src/csplit/args.v create mode 100644 src/csplit/csplit.v create mode 100644 src/csplit/csplit_test.v create mode 100644 src/csplit/ere.v create mode 100644 src/csplit/ere_parser.v diff --git a/src/csplit/args.v b/src/csplit/args.v new file mode 100644 index 00000000..8989644c --- /dev/null +++ b/src/csplit/args.v @@ -0,0 +1,246 @@ +module main + +import common +import os + +// quoted wraps an operand the way GNU does. The quotes are U+2018 and U+2019, and +// a backslash inside them is doubled: the operand /a\bb/ is reported as ‘/a\\bb/’, +// which is checked by counting bytes rather than by reading it. +fn quoted(text string) string { + return quote_left + text.replace('\\', '\\\\') + quote_right +} + +// parse_args turns the operand list into patterns, in the order GNU classifies them: +// while it walks the arguments. That is why a bad pattern is reported before a file +// that cannot be opened, and why "ten.txt /3/ nosuch" complains about "nosuch" being +// an invalid pattern rather than about a second operand. +fn parse_args() !Settings { + mut fp := common.flag_parser(os.args) + fp.application(app_name) + fp.description('Split a file into pieces separated by pattern lines.') + + // A repeated -f or -b takes the last one, which is measured: "-f xx -f yy" names + // its pieces yy00 and yy01. The flag module keeps the first, so these are read as + // multi-values and the last is taken here. + prefixes := fp.string_multi('prefix', `f`, 'use PREFIX instead of ‘xx’') + suffixes := fp.string_multi('suffix-format', `b`, 'use sprintf FORMAT instead of %02d') + digits_text := fp.string('digits', `n`, default_digits.str(), + 'use specified number of digits instead of 2') + keep_files := fp.bool('keep-files', `k`, false, 'do not remove output files on errors') + suppress := fp.bool('suppress-matched', 0, false, 'suppress the lines matching PATTERN') + silent := fp.bool('quiet', `s`, false, 'do not print counts of output file sizes') + elide := fp.bool('elide-empty-files', `z`, false, 'suppress empty output files') + + rest := fp.remaining_parameters() + if rest.len < 2 { + // GNU names the operand it ran out of patterns after, and unlike a complaint + // about a pattern it is the only one that does not arrive through the shared + // helper, because that helper adds a "Try --help" line csplit never prints. + last := if rest.len > 0 { rest[rest.len - 1] } else { '' } + // This one complaint does carry the advice line, which is the other half of + // why it is not routed through the shared helper: the helper would add that + // line to every operand complaint, and csplit prints it only for this one. + operand_error("missing operand after ${quoted(last)}\nTry '${app_name} --help' for more information.") + } + mut set := Settings{ + file: rest[0] + prefix: default_prefix + suffix: '' + keep_files: keep_files + suppress: suppress + silent: silent + elide_empty: elide + } + if prefixes.len > 0 { + set.prefix = prefixes[prefixes.len - 1] + } + if suffixes.len > 0 { + set.suffix = suffixes[suffixes.len - 1] + } + set.digits = parse_digits(digits_text)! + for arg in rest[1..] { + set.patterns << parse_pattern(arg)! + } + return set +} + +// check_repetitions refuses a {INTEGER} or {*} that has no plain pattern in front of +// it, or that follows another repetition. +// +// This runs after the input has been opened rather than with the rest of the +// arguments, which is measured: given a first operand of "3" and a pattern of "{1}", +// GNU complains that it cannot open "3" and never mentions the repetition. The check +// is still before anything is written, so "/3/ {2} {2}" over ten lines prints nothing +// at all, where a check during the run would already have printed 4 and 17. +fn check_repetitions(pats []Pat) { + mut have_plain := false + mut repeats := false + for p in pats { + if p.kind != .repeat { + have_plain = true + repeats = false + continue + } + if !have_plain || repeats { + operand_error(quoted(p.text) + ': invalid pattern') + } + repeats = true + } +} + +// operand_error reports a complaint about an operand. It prints the message on its +// own rather than through common.exit_with_error_message, because that helper adds a +// "Try 'csplit --help' for more information." line that csplit does not print for +// anything it says about its operands. +@[noreturn] +fn operand_error(message string) { + eprintln('${app_name}: ${message}') + exit(1) +} + +// parse_digits reads -n. GNU takes 0 and refuses a negative one, and says so in its +// own words, which is why this is not left to the flag module's integer reader. +fn parse_digits(text string) !int { + mut value := 0 + mut seen := false + for b in text { + if b < `0` || b > `9` { + // A leading minus is the one non-digit GNU names separately. + if text.len > 1 && text[0] == `-` && all_digits(text[1..]) { + return error('invalid number: ‘${text}’: Numerical result out of range') + } + return error('invalid number: ‘${text}’') + } + value = value * 10 + int(b - `0`) + seen = true + } + if !seen { + return error('invalid number: ‘${text}’') + } + return value +} + +fn all_digits(s string) bool { + if s.len == 0 { + return false + } + for b in s { + if b < `0` || b > `9` { + return false + } + } + return true +} + +// parse_pattern classifies one PATTERN operand. The delimiter is found at the first +// occurrence of its own character and is not itself escapable, which is measured: +// "/b\/" is the pattern "b\" followed by nothing, and csplit refuses it as a trailing +// backslash rather than reading the slash as part of the pattern. +fn parse_pattern(arg string) !Pat { + if arg.len > 0 && arg[0] == `/` { + return parse_delimited(arg, `/`, .regex) + } + if arg.len > 0 && arg[0] == `%` { + return parse_delimited(arg, `%`, .skip) + } + if arg.len > 2 && arg[0] == `{` && arg[arg.len - 1] == `}` { + inner := arg[1..arg.len - 1] + if inner == '*' { + return Pat{ + kind: .repeat + text: arg + number: -1 + } + } + if all_digits(inner) { + return Pat{ + kind: .repeat + text: arg + number: inner.int() + } + } + return error(quoted(arg) + ': invalid pattern') + } + if arg.len > 0 && (arg[0] == `+` || (arg[0] >= `0` && arg[0] <= `9`)) { + // "+3" is accepted and means 3. A bare 0 is refused by name rather than + // being taken for the start of the file. + body := if arg[0] == `+` { arg[1..] } else { arg } + if all_digits(body) { + number := body.int() + if number == 0 { + // The operand is echoed as it was written, so "+0" is named rather + // than the 0 it stands for. + return error('${arg}: line number must be greater than zero') + } + return Pat{ + kind: .line_number + text: arg + number: number + } + } + } + return error(quoted(arg) + ': invalid pattern') +} + +// parse_delimited reads a /REGEXP/[OFFSET] or %REGEXP%[OFFSET] operand. +fn parse_delimited(arg string, delim u8, kind PatKind) !Pat { + mut close := -1 + for i in 1 .. arg.len { + if arg[i] == delim { + close = i + break + } + } + if close < 0 { + // The complaint names the delimiter that was opened, with plain quotes, and + // the operand outside any: "/8%: closing delimiter '/' missing". + return error("${arg}: closing delimiter '${delim.ascii_str()}' missing") + } + source := arg[1..close] + rest := arg[close + 1..] + mut offset := 0 + if rest.len > 0 { + offset = parse_offset(arg, rest) or { return err } + } + return Pat{ + kind: kind + text: arg + re: source + offset: offset + } +} + +// parse_offset reads the number after the closing delimiter. It is optional, but +// anything there has to be a number: "/b/ " is refused where "/b/" is not. +fn parse_offset(arg string, rest string) !int { + mut negative := false + mut body := rest + if rest[0] == `+` || rest[0] == `-` { + negative = rest[0] == `-` + body = rest[1..] + } + if !all_digits(body) { + return error(quoted(arg) + ': integer expected after delimiter') + } + return if negative { -body.int() } else { body.int() } +} + +// read_input reads the whole operand up front. csplit reports the size of every piece +// in bytes and cuts at line boundaries, so there is no streaming case to keep. +fn read_input(path string) ![]u8 { + if path == '-' { + mut out := []u8{} + for { + line := os.get_raw_line() + if line.len == 0 { + break + } + out << line.bytes() + } + return out + } + if !os.exists(path) { + return error("cannot open '${path}' for reading: No such file or directory") + } + return os.read_bytes(path)! +} diff --git a/src/csplit/csplit.v b/src/csplit/csplit.v new file mode 100644 index 00000000..58be21f2 --- /dev/null +++ b/src/csplit/csplit.v @@ -0,0 +1,513 @@ +module main + +import os + +const app_name = 'csplit' +const default_prefix = 'xx' +const default_digits = 2 +// GNU quotes the operand in most of its complaints with these, not with `'`. +const quote_left = '\u2018' +const quote_right = '\u2019' + +// PatKind is the five forms csplit --help lists for a PATTERN. +enum PatKind { + // INTEGER: copy up to but not including that line number. + line_number + // /REGEXP/[OFFSET]: copy up to but not including a matching line. + regex + // %REGEXP%[OFFSET]: skip to, but not including, a matching line. + skip + // {INTEGER} and {*}: repeat the pattern before it. + repeat +} + +struct Pat { +mut: + kind PatKind + // text is the operand as the user wrote it, because the diagnostics quote the + // operand rather than a normalised form of it. + text string + // number is the line number for .line_number, or the repeat count for + // .repeat, where -1 stands for {*}. + number int + // re is the BRE source for .regex and .skip. + re string + // offset shifts the matched line, and is 0 when the operand carried none. + offset int +} + +// A Pat with its regular expression already built, so a pattern applied many times +// over does not rebuild it each time. +struct Compiled { +mut: + pat Pat + ere Ere +} + +struct Settings { +mut: + file string + patterns []Pat + prefix string = default_prefix + suffix string + digits int = default_digits + keep_files bool + suppress bool + silent bool + elide_empty bool +} + +// LineIndex records where each line starts and ends, so a pattern can name a line +// number and a piece can be cut at one. Lines are counted from 1, and the trailing +// newline belongs to the line it ends. +struct LineIndex { +mut: + starts []int + // ends is one past the last byte of a line, newline included. + ends []int +} + +// LineIndex.build walks the data once. A last line without a newline is still a +// line and GNU writes it: over "1\n2\n3" a split at line 2 gives "1\n" and "2\n3", +// so three lines, not two. +fn LineIndex.build(data []u8) LineIndex { + mut li := LineIndex{} + mut at := 0 + for at < data.len { + li.starts << at + mut stop := at + for stop < data.len && data[stop] != `\n` { + stop++ + } + if stop < data.len { + stop++ + } + li.ends << stop + at = stop + } + return li +} + +// subject is the line as a pattern sees it: the newline is not part of it, because +// GNU anchors $ to the end of the line. A CR is part of it, which is why "^2$" does +// not match the second line of a CRLF file. +fn (li LineIndex) subject(data []u8, line int) []u8 { + mut stop := li.ends[line - 1] + if stop > li.starts[line - 1] && data[stop - 1] == `\n` { + stop-- + } + return data[li.starts[line - 1]..stop] +} + +fn (li LineIndex) count() int { + return li.starts.len +} + +// Cutter holds what one run of csplit needs, so the pattern loop does not have to +// thread five values through every call. +struct Cutter { +mut: + settings Settings + patterns []Compiled + data []u8 + lines LineIndex + // at is where the piece being built starts in data. + at int + // after is the line a pattern must match later than. It starts at 0, which is + // why a first pattern may match the very first line, and it rises to the line + // just cut at, which is why "/5/ /5/" finds no match the second time. + after int + // previous is the pattern a repetition repeats, kept because the repetition is a + // separate operand that names nothing itself. + previous Compiled + // skipped records that the last successful pattern was a % skip, and pending_skip + // that the pattern now failing is one. fail reads both; see its comment. + skipped bool + pending_skip bool + // quiet_failure suppresses the remainder for the one failure that reports nothing + // at all, which is a line number against an empty input. + quiet_failure bool + index int + written []string +} + +fn main() { + settings := parse_args() or { operand_error(err.msg()) } + // The patterns are compiled before the input is opened, because GNU reads its + // whole operand list before it touches the file: "/99/ [[:bogus:]]" over ten + // lines complains about the class name and prints nothing, where a complaint + // during the run would already have printed 21. + patterns := compile_all(settings.patterns) + data := read_input(settings.file) or { + eprintln('${app_name}: ${err.msg()}') + exit(1) + } + check_repetitions(settings.patterns) + mut c := Cutter{ + settings: settings + patterns: patterns + data: data + lines: LineIndex.build(data) + } + c.execute() +} + +// compile_all builds every pattern's regular expression up front, and gives up on the +// first one that will not compile. +fn compile_all(pats []Pat) []Compiled { + mut out := []Compiled{cap: pats.len} + for p in pats { + mut c := Compiled{ + pat: p + } + if p.kind == .regex || p.kind == .skip { + c.ere.compile(p.re.bytes()) or { + operand_error('${quoted(p.text)}: invalid regular expression: ${err.msg()}') + } + } + out << c + } + return out +} + +// execute walks the pattern list, writing a piece at every cut. It reports its own +// failures and leaves, because a failure has to remove the pieces already written +// unless -k was given, and a returned error cannot carry that decision. +fn (mut c Cutter) execute() { + mut i := 0 + for i < c.patterns.len { + cur := c.patterns[i] + if cur.pat.kind == .repeat { + c.repeat(c.previous, cur) or { c.fail(err.msg(), true) } + i++ + continue + } + // A {*} is handled together with the pattern it repeats, because it decides + // whether that pattern's own first failure matters, and because it ends the + // list: "/3/ {*} /6/" over ten lines cuts only at line 3. + if i + 1 < c.patterns.len && c.patterns[i + 1].pat.kind == .repeat + && c.patterns[i + 1].pat.number < 0 { + if cur.pat.kind == .skip { + // A % skip followed by {*} writes nothing at all and reports no size, + // which is what "twenty lines, %1% {*}" does. Reproduced rather than + // tidied, because matching GNU is the point of the port. + return + } + c.star(cur) or { c.fail(err.msg(), false) } + c.write_piece(c.at, c.data.len) or { c.fail(err.msg(), false) } + return + } + c.apply(cur, 0) or { c.fail(err.msg(), false) } + c.previous = cur + i++ + } + // Whatever is left after the last cut is the last piece. + c.write_piece(c.at, c.data.len) or { c.fail(err.msg(), false) } +} + +// star applies a pattern and keeps applying it for as long as it still matches. A +// {N} repetition is not the same thing and is handled by repeat. +// +// {*} absorbs the first failure of a pattern that searches, where {N} does not: +// over ten lines "/99/ {*}" succeeds and writes the file whole, while "/99/ {2}" +// fails. A line number gets no such softening, so "99 {*}" is still out of range. +// +// Nothing reads the position afterwards, because a {*} ends the pattern list, so +// there is no "where the repetition left off" to record here. +fn (mut c Cutter) star(p Compiled) ! { + mut done := 0 + for { + cut := c.find_cut(p, done) or { + if p.pat.kind != .line_number { + return + } + return err + } + c.cut_at(cut, false) or { return err } + done++ + } +} + +// repeat applies a pattern N more times after its first use. The first use has +// already happened in execute, so every failure here names a repetition number. +fn (mut c Cutter) repeat(previous Compiled, spec Compiled) ! { + mut done := 0 + mut last := 0 + for done < spec.pat.number { + cut := c.find_cut(previous, done + 1) or { return err } + c.cut_at(cut, previous.pat.kind == .skip) or { return err } + last = cut + done++ + } + // The line a repetition ends on is one before the line it last cut at, which is + // measured: "3 {2} /9/" over ten lines finds line 9 again and writes an empty + // piece, while "3 {1} /4/" finds nothing, because its last cut was line 6 and + // line 4 is behind both 6 and 5. + c.after = if last > 1 { last - 1 } else { 0 } +} + +// apply uses one pattern once and writes the piece it ends. rep is the 1-based +// repetition number, or 0 when this is not a repetition; it only changes the wording +// of a failure. +fn (mut c Cutter) apply(p Compiled, rep int) ! { + cut := c.find_cut(p, rep) or { return err } + c.cut_at(cut, p.pat.kind == .skip) or { return err } +} + +// find_cut reports which line a pattern cuts before. It is all of a pattern's +// behaviour; everything else in this file is bookkeeping around the answer. +fn (mut c Cutter) find_cut(p Compiled, rep int) !int { + // fail reads this to tell a % pattern that failed from any other, and cut_at + // reads the other one to tell a % pattern that succeeded. + c.pending_skip = p.pat.kind == .skip + mut complaint := '' + if rep > 0 { + complaint = ' on repetition ${rep}' + } + if p.pat.kind == .line_number { + // A line number against an empty input is refused with a complaint of its + // own, and GNU still leaves an empty first piece behind without reporting its + // size. Measured on a zero-byte file: "csplit: input disappeared", one empty + // xx00, nothing on stdout, exit 1. + if c.lines.count() == 0 { + c.write_empty_piece() + c.quiet_failure = true + return error('input disappeared') + } + // A line number is absolute the first time and steps by its own value on a + // repetition: "3 {2}" over ten lines cuts at 3, 6 and 9, while "2" alone + // cuts at 2. + mut want := p.pat.number + if rep > 0 { + want = c.after + p.pat.number + } + if want > c.lines.count() { + return error(quoted(p.pat.text) + ': line number out of range' + complaint) + } + return want + } + found := c.search(p) or { + return error(quoted(p.pat.text) + ': match not found' + complaint) + } + return c.shift(found, p) or { return err } +} + +// search finds the first line after the current position that the pattern matches. +// A pattern matching only the line it is already sitting on counts as no match, +// which is what makes "/5/ /5/" fail and what stops {*} for ever. +fn (mut c Cutter) search(p Compiled) ?int { + mut line := c.after + 1 + for line <= c.lines.count() { + if p.ere.matches(c.lines.subject(c.data, line)) { + return line + } + line++ + } + return none +} + +// shift moves a match by the operand's OFFSET. An offset landing outside the file is +// a range complaint rather than a failed search: "/3/+99" over ten lines says "line +// number out of range", not "match not found". +fn (mut c Cutter) shift(found int, p Compiled) !int { + if p.pat.offset == 0 { + return found + } + mut want := found + p.pat.offset + if want < 1 || want > c.lines.count() { + return error(quoted(p.pat.text) + ': line number out of range') + } + return want +} + +// cut_at writes the piece that ends at the given line and moves the position on. +// +// The piece always stops at the *start* of the cut line, so the matched line belongs +// to the piece that follows. --suppress-matched does not shorten the piece being +// written; it drops the matched line so that the next one starts after it, which is +// the difference measured over ten lines between "/5/" giving 8 and 13 bytes and +// "--suppress-matched /5/" giving 8 and 11. +// +// A .skip pattern writes nothing at all: it moves the start of the current piece up to +// its match and throws away what was before it. That is the whole difference between +// "/5/" and "%5%": over ten lines the first gives pieces of 8 and 13 bytes and the +// second a single piece of 13. +fn (mut c Cutter) cut_at(line int, skipping bool) ! { + mut next := c.lines.starts[line - 1] + if c.settings.suppress { + next = c.lines.ends[line - 1] + } + if skipping { + c.at = next + c.after = line + c.skipped = true + return + } + c.write_piece(c.at, c.lines.starts[line - 1]) or { return err } + // The position never goes backwards, so a cut naming a line that has gone by + // leaves the piece where it already is. + if next > c.at { + c.at = next + } + c.after = line + c.skipped = false +} + +// fail writes out the piece that was in progress, prints the failure, removes the +// pieces already written unless -k was given, and leaves with GNU's exit code. +// +// The order is measured rather than tidied: "/99/" over ten lines prints 21 and then +// fails, and "/0/ /5/" prints 18 and 3 before failing, so the remainder is written and +// counted before the complaint is printed. +// +// Two measured cases write no remainder, and both are about a % skip. A pattern that +// failed while looking for its match writes nothing: "/3/ %99%" over ten lines prints +// only 4, where "%3% /99/" prints 17. And a repetition that failed straight after a +// skip writes nothing either: "%3% {1}" prints nothing at all, where "/3/ {1}" prints +// 4 and 17. +fn (mut c Cutter) fail(message string, from_repeat bool) { + if !(c.quiet_failure || c.pending_skip || (c.skipped && from_repeat)) { + c.write_piece(c.at, c.data.len) or {} + } + c.die(message) +} + +fn (mut c Cutter) die(message string) { + eprintln('${app_name}: ${message}') + if !c.settings.keep_files { + c.remove_written() + } + exit(1) +} + +fn (mut c Cutter) remove_written() { + for name in c.written { + os.rm(name) or {} + } +} + +// write_piece writes one output file and prints its size, which is what csplit +// reports on stdout unless -s was given. +fn (mut c Cutter) write_piece(from int, to int) ! { + // A cut can land behind the piece already built, which happens when a line + // number names a line that has gone by: "3 {2} 7" over ten lines cuts at 9 and + // then at 7, and GNU answers the second with an empty piece. + mut end := to + if end < from { + end = from + } + if c.settings.elide_empty && end == from { + return + } + name := c.piece_name() or { return err } + mut f := os.create(name) or { return error('${name}: ${os.error_posix()}') } + f.write(c.data[from..end]) or { + f.close() + return error('${name}: ${os.error_posix()}') + } + f.close() + c.written << name + if !c.settings.silent { + println(end - from) + } + c.index++ +} + +// write_empty_piece creates the first output file with nothing in it and says nothing +// about its size. Only the empty-input case reaches it. +// +// It is deliberately not recorded as written, because the cleanup on a failure removes +// the pieces that were and GNU leaves this one behind even without -k: it never had a +// byte put in it. +fn (mut c Cutter) write_empty_piece() { + name := c.piece_name() or { return } + mut f := os.create(name) or { return } + f.close() +} + +// piece_name builds PREFIX followed by the piece number. A -b format replaces the +// default one and makes -n irrelevant, which is measured: "-b %02d -n 3" over ten +// lines still writes xx00 and xx01. +fn (c Cutter) piece_name() !string { + return c.settings.prefix + suffix_digits(c.settings.suffix, c.settings.digits, + c.index) or { return err } +} + +// suffix_digits applies the -b format, or the "%0Nd" that -n asks for when -b was not +// given. Only one conversion is allowed and it has to be d, because that is all GNU +// accepts: "%s" and "%ld" are refused by name and "%d%d" is refused for having two. +fn suffix_digits(format string, digits int, index int) !string { + if format == '' { + return pad(index.str(), digits, true, false) + } + mut at := -1 + for i, b in format { + if b == `%` { + if at != -1 { + return error('too many % conversion specifications in suffix') + } + at = i + } + } + if at == -1 { + return error('missing % conversion specification in suffix') + } + mut i := at + 1 + mut left := false + mut zero := false + for i < format.len && (format[i] == `-` || format[i] == `+` || format[i] == ` ` + || format[i] == `#` || format[i] == `0`) { + if format[i] == `-` { + left = true + } + if format[i] == `0` { + zero = true + } + i++ + } + mut width := 0 + for i < format.len && format[i] >= `0` && format[i] <= `9` { + width = width * 10 + int(format[i] - `0`) + i++ + } + mut precision := -1 + if i < format.len && format[i] == `.` { + i++ + precision = 0 + for i < format.len && format[i] >= `0` && format[i] <= `9` { + precision = precision * 10 + int(format[i] - `0`) + i++ + } + } + if i >= format.len { + return error('missing % conversion specification in suffix') + } + if format[i] != `d` { + return error('invalid conversion specifier in suffix: ' + + format[i].ascii_str()) + } + // A precision on d is a minimum of digits rather than a column, so "%.2d" of 0 is + // "00" and not " 0", and it zero-fills whatever the flags say. + mut out := index.str() + if precision >= 0 { + out = pad(out, precision, true, false) + } + // The text either side of the conversion is kept, which is what makes + // "-b out%d" name its pieces xxout0 and xxout1. + return format[..at] + pad(out, width, zero, left) + format[i + 1..] +} + +// pad applies printf's width rules to a non-negative number, which is the only thing +// a piece number ever is. +fn pad(digits string, want int, zero bool, left bool) string { + if digits.len >= want { + return digits + } + if left { + return digits + ` `.str().repeat(want - digits.len) + } + if zero { + return `0`.str().repeat(want - digits.len) + digits + } + return ` `.str().repeat(want - digits.len) + digits +} diff --git a/src/csplit/csplit_test.v b/src/csplit/csplit_test.v new file mode 100644 index 00000000..488f2363 --- /dev/null +++ b/src/csplit/csplit_test.v @@ -0,0 +1,507 @@ +import common.testing +import os + +const rig = testing.prepare_rig(util: 'csplit') +const exe = rig.executable_under_test + +// ten is ten lines of one, two and three digit numbers, 21 bytes: two for each of +// 1..9 and three for 10. +const input_name = 'in.txt' +const stdin_name = 'stdin.txt' +const ten = '1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n' +// A file whose third line is "12", so a pattern of "2" and a line number of 2 have to +// be told apart: the text "2" is on line 3 and the second line is "b". +const ambiguous = 'a\nb\n12\nc\n' +// Nine lines of assorted text, 47 bytes, with two empty ones. +const words = 'alpha\n# comment\n\nbeta\ngamma\nEND\ndelta\n\nepsilon\n' + +// Run is one invocation of the binary under test. +struct Run { +mut: + code int + stdout string + stderr string + // names and contents hold the pieces in name order, which is the order csplit + // creates them in. + names []string + contents []string +} + +// Case is one operand list and the complaint csplit is expected to make about it. A +// list of these rather than a map, because a map key cannot be an array. +struct Case { + args []string + complaint string +} + +// run drives the binary with an argument array and reads back what it wrote. +// +// os.execute is deprecated, goes through a shell, and cannot give a child any stdin at +// all; os.new_process takes the arguments as given and can read the child's stdin from +// a file, which is the only way an operand of "-" can be reached from a test. +fn run(args []string, stdin_path string) Run { + mut p := os.new_process(exe) + p.set_args(args) + p.set_redirect_stdio() + if stdin_path != '' { + p.set_stdin_path(stdin_path) + } + p.wait() + mut r := Run{ + code: p.code + stdout: p.stdout_slurp() + stderr: p.stderr_slurp() + } + p.close() + r.collect() + return r +} + +// run_in runs the tool over the input file, which is what nearly every case does, and +// saves each test from naming it. +fn run_in(args []string) Run { + mut full := [input_name] + full << args + return run(full, '') +} + +// collect reads the pieces out of the working directory, which prepare_rig has already +// moved into the tool's own temporary folder. +fn (mut r Run) collect() { + mut names := (os.ls('.') or { [] }).filter(it != input_name && it != stdin_name).sorted() + for n in names { + r.names << n + r.contents << os.read_file(n) or { '' } + } +} + +// sizes reads back just the byte counts csplit printed, which is the shape most of its +// behaviour is about. +fn (r Run) sizes() []string { + return r.stdout.trim_space().split_into_lines().filter(it != '') +} + +// expect is the assertion used everywhere: the exit code, the pieces and their names, +// their contents, and what csplit printed. +fn (r Run) expect(code int, names []string, contents []string, sizes []string) { + assert r.code == code, 'exit ${r.code}, stderr [${r.stderr}]' + assert r.names == names, 'names ${r.names}' + assert r.contents == contents, 'contents ${r.contents}' + assert r.sizes() == sizes, 'stdout [${r.stdout}]' +} + +// expect_says checks only the complaint, for the cases where the pieces are not the +// point. The slurped streams keep the newline they were written with, so it is trimmed +// rather than spelled out in every expectation. +fn (r Run) expect_says(code int, complaint string) { + assert r.code == code, 'exit ${r.code}, stderr [${r.stderr}]' + assert r.stderr.trim_space() == complaint, 'said [${r.stderr}]' +} + +// reset removes what a previous test left behind. +fn reset() { + for f in os.ls('.') or { [] } { + if f != input_name && f != stdin_name { + os.rm(f) or {} + } + } +} + +fn write_input(content string) { + reset() + os.write_file(input_name, content) or { panic(err) } +} + +fn test_help_and_version() { + rig.assert_help_and_version_options_work() +} + +// A bare number is a line number, not the text it spells. Over a file whose third line +// is "12", "2" cuts before line 2 and "/2/" cuts before line 3. +fn test_bare_number_is_a_line_number_not_a_text_match() { + write_input(ambiguous) + run_in(['2']).expect(0, ['xx00', 'xx01'], ['a\n', 'b\n12\nc\n'], ['2', '7']) + reset() + run_in(['/2/']).expect(0, ['xx00', 'xx01'], ['a\nb\n', '12\nc\n'], ['4', '5']) +} + +// The matched line belongs to the piece that follows, not the one that ends. +fn test_line_number_cuts_before_that_line() { + write_input(ten) + run_in(['3']).expect(0, ['xx00', 'xx01'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) +} + +fn test_two_patterns() { + write_input(ten) + run_in(['/3/', '/6/']).expect(0, ['xx00', 'xx01', 'xx02'], ['1\n2\n', '3\n4\n5\n', '6\n7\n8\n9\n10\n'], [ + '4', + '6', + '11', + ]) +} + +// --suppress-matched drops the matched line, so the piece after it starts one line +// further on. +fn test_suppress_matched_drops_the_line() { + write_input(ten) + run_in(['--suppress-matched', '/5/']).expect(0, ['xx00', 'xx01'], + ['1\n2\n3\n4\n', '6\n7\n8\n9\n10\n'], ['8', '11']) + reset() + run_in(['--suppress-matched', '5']).expect(0, ['xx00', 'xx01'], + ['1\n2\n3\n4\n', '6\n7\n8\n9\n10\n'], ['8', '11']) + reset() + run_in(['--suppress-matched', '%5%']).expect(0, ['xx00'], ['6\n7\n8\n9\n10\n'], + ['11']) +} + +// A failure prints the piece it had reached before complaining, removes the pieces +// again, and leaves with 1. The order is checked because the obvious one, complaining +// first, would print nothing. +fn test_no_match_prints_the_remainder_then_fails() { + write_input(ten) + run_in(['/99/']).expect(1, []string{}, []string{}, ['21']) + reset() + run_in(['/99/', '-k']).expect(1, ['xx00'], [ten], ['21']) + reset() + run_in(['99']).expect(1, []string{}, []string{}, ['21']) +} + +fn test_no_match_says_which_pattern() { + write_input(ten) + run_in(['/99/']).expect_says(1, 'csplit: ‘/99/’: match not found') + reset() + run_in(['99']).expect_says(1, 'csplit: ‘99’: line number out of range') + reset() + run_in(['99', '-k']).expect_says(1, 'csplit: ‘99’: line number out of range') +} + +fn test_quiet_prints_no_sizes() { + write_input(ten) + run_in(['-s', '/5/']).expect(0, ['xx00', 'xx01'], ['1\n2\n3\n4\n', '5\n6\n7\n8\n9\n10\n'], []string{}) + reset() + run_in(['--quiet', '/5/']).expect(0, ['xx00', 'xx01'], ['1\n2\n3\n4\n', '5\n6\n7\n8\n9\n10\n'], []string{}) +} + +// -f given twice takes the last. %REGEXP% throws away everything before its match +// rather than writing it. +fn test_prefix_takes_the_last_and_skip_discards() { + write_input(ten) + run_in(['-f', 'xx', '-f', 'yy', '/3/']).expect(0, ['yy00', 'yy01'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) + reset() + run_in(['%5%']).expect(0, ['xx00'], ['5\n6\n7\n8\n9\n10\n'], ['13']) + reset() + run_in(['%5%', '/8/']).expect(0, ['xx00', 'xx01'], ['5\n6\n7\n', '8\n9\n10\n'], + ['6', '7']) +} + +fn test_digits_and_suffix_format() { + write_input(ten) + run_in(['-n', '3', '/3/', '/6/']).expect(0, ['xx000', 'xx001', 'xx002'], + ['1\n2\n', '3\n4\n5\n', '6\n7\n8\n9\n10\n'], ['4', '6', '11']) + reset() + // A width of 0 pads to nothing at all. + run_in(['-n', '0', '/3/']).expect(0, ['xx0', 'xx1'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) + reset() + // -b keeps the prefix and puts the formatted number after it. + run_in(['-b', 'out%d', '/3/']).expect(0, ['xxout0', 'xxout1'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) + reset() + // -b wins over -n, which is measured rather than assumed. + run_in(['-b', '%02d', '-n', '3', '/3/']).expect(0, ['xx00', 'xx01'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) +} + +fn test_suffix_format_complaints() { + for format, complaint in { + 'nopct': 'csplit: missing % conversion specification in suffix' + 'x%s': 'csplit: invalid conversion specifier in suffix: s' + '%ld': 'csplit: invalid conversion specifier in suffix: l' + '%d%d': 'csplit: too many % conversion specifications in suffix' + } { + write_input(ten) + run_in(['-b', format, '/3/']).expect_says(1, complaint) + } +} + +fn test_elide_empty_files() { + write_input(ten) + // The two patterns match lines 5 and 6, so without --suppress-matched nothing is + // empty and -z changes nothing. + run_in(['-z', '/5/', '/6/']).expect(0, ['xx00', 'xx01', 'xx02'], + ['1\n2\n3\n4\n', '5\n', '6\n7\n8\n9\n10\n'], ['8', '2', '11']) + reset() + // With it, the middle piece would be empty, and -z neither writes nor counts it. + run_in(['-z', '--suppress-matched', '/5/', '/6/']).expect(0, ['xx00', 'xx01'], + ['1\n2\n3\n4\n', '7\n8\n9\n10\n'], ['8', '9']) + reset() + run_in(['--suppress-matched', '/5/', '/6/']).expect(0, ['xx00', 'xx01', 'xx02'], + ['1\n2\n3\n4\n', '', '7\n8\n9\n10\n'], ['8', '0', '9']) +} + +// The offset after the closing delimiter moves the cut, and one that lands past the end +// of the file is a range complaint rather than a failed search. +fn test_offset_after_the_delimiter() { + write_input(ten) + run_in(['/3/+1']).expect(0, ['xx00', 'xx01'], ['1\n2\n3\n', '4\n5\n6\n7\n8\n9\n10\n'], [ + '6', + '15', + ]) + reset() + run_in(['/3/-1']).expect(0, ['xx00', 'xx01'], ['1\n', '2\n3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '2', + '19', + ]) + reset() + run_in(['/3/+99']).expect_says(1, 'csplit: ‘/3/+99’: line number out of range') +} + +// A line number repeated steps by its own value, which is why "3 {2}" cuts at 3, 6 and +// 9 rather than at 3 three times. +fn test_repeating_a_line_number() { + write_input(ten) + run_in(['3', '{2}']).expect(0, ['xx00', 'xx01', 'xx02', 'xx03'], ['1\n2\n', '3\n4\n5\n', '6\n7\n8\n', + '9\n10\n'], ['4', '6', '6', '5']) + reset() + // {*} has no soft ending for a line number, so running off the end is a failure, and + // it is reported as repetition 3 because the first two repeats did fit. + run_in(['3', '{*}']).expect_says(1, + 'csplit: ‘3’: line number out of range on repetition 3') + reset() + run_in(['3', '{2}', '/9/']).expect(0, ['xx00', 'xx01', 'xx02', 'xx03', 'xx04'], + ['1\n2\n', '3\n4\n5\n', '6\n7\n8\n', '', '9\n10\n'], ['4', '6', '6', '0', '5']) +} + +fn test_repeating_a_pattern_until_it_stops_matching() { + write_input(ten) + run_in(['/3/', '{*}']).expect(0, ['xx00', 'xx01'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) + reset() + // {*} absorbs the first failure of a pattern that searches, where {2} does not. + run_in(['/99/', '{*}']).expect(0, ['xx00'], [ten], ['21']) + reset() + run_in(['/99/', '{2}']).expect_says(1, 'csplit: ‘/99/’: match not found') + reset() + // {*} ends the pattern list: the /6/ after it never runs. + run_in(['/3/', '{*}', '/6/']).expect(0, ['xx00', 'xx01'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) +} + +fn test_a_repetition_must_follow_a_plain_pattern() { + for c in [ + Case{ + args: ['{2}'] + complaint: 'csplit: ‘{2}’: invalid pattern' + }, + Case{ + args: ['{*}'] + complaint: 'csplit: ‘{*}’: invalid pattern' + }, + Case{ + args: ['/3/', '{2}', '{2}'] + complaint: 'csplit: ‘{2}’: invalid pattern' + }, + ] { + write_input(ten) + run_in(c.args).expect_says(1, c.complaint) + } +} + +fn test_bad_operands() { + for c in [ + Case{ + args: ['0'] + complaint: 'csplit: 0: line number must be greater than zero' + }, + Case{ + args: ['+0'] + complaint: 'csplit: +0: line number must be greater than zero' + }, + Case{ + args: ['zzz'] + complaint: 'csplit: ‘zzz’: invalid pattern' + }, + Case{ + args: ['%'] + complaint: "csplit: %: closing delimiter '%' missing" + }, + Case{ + args: ['/8%'] + complaint: "csplit: /8%: closing delimiter '/' missing" + }, + Case{ + args: ['/b/I'] + complaint: 'csplit: ‘/b/I’: integer expected after delimiter' + }, + ] { + write_input(ten) + run_in(c.args).expect_says(1, c.complaint) + } +} + +// A pattern is compiled while the operands are read, so a bad one is refused before +// anything is opened or written. The expected complaints double their backslashes, +// because csplit quotes the operand the way GNU does. +fn test_a_pattern_is_compiled_before_anything_is_written() { + for c in [ + Case{ + args: ['/[[:bogus:]]/'] + complaint: 'csplit: ‘/[[:bogus:]]/’: invalid regular expression: Invalid character class name' + }, + Case{ + args: ['/^\\)/'] + complaint: 'csplit: ‘/^\\\\)/’: invalid regular expression: Unmatched ) or \\)' + }, + Case{ + args: ['/^\\101$/'] + complaint: 'csplit: ‘/^\\\\101$/’: invalid regular expression: Invalid back reference' + }, + Case{ + args: ['/1\\{/'] + complaint: 'csplit: ‘/1\\\\{/’: invalid regular expression: Unmatched \\{' + }, + ] { + write_input(ten) + r := run_in(c.args) + r.expect_says(1, c.complaint) + assert r.stdout == '', '${c.args} printed [${r.stdout}]' + assert r.names == []string{}, '${c.args} left ${r.names}' + } +} + +// The dialect is a basic regular expression, not the extended one the manual's word +// "REGEXP" suggests. Each expectation below is what GNU does with the same pattern over +// the same input, because the two dialects disagree about most of these. +fn test_the_pattern_language_is_a_basic_regular_expression() { + write_input(words) + // + is an ordinary character, so this wants a literal plus at the end. It matches + // the fourth line, "beta", so the first piece is the three lines above it. + run_in(['/^beta\\+$/']).expect(0, ['xx00', 'xx01'], + ['alpha\n# comment\n\n', 'beta\ngamma\nEND\ndelta\n\nepsilon\n'], ['17', '30']) + reset() + // So are the brackets, which is why this wants the text "(alpha)" and does not + // find it. The closing \) with nothing to open it is refused outright. + run_in(['/^(alpha)$/']).expect_says(1, 'csplit: ‘/^(alpha)$/’: match not found') + reset() + run_in(['/^(alpha\\|beta\\)$/']).expect_says(1, + 'csplit: ‘/^(alpha\\\\|beta\\\\)$/’: invalid regular expression: Unmatched ) or \\)') + reset() + // The escaped forms are the operators, and the bar splits the pattern whole, so a + // ^ after it still anchors. + run_in(['/^e\\|^a/']).expect(0, ['xx00', 'xx01'], ['', words], ['0', '47']) + reset() + // A counted interval needs the escaped braces, and both bounds take one. Each of + // these matches the first line, so the first piece is empty. + run_in(['/^[[:alpha:]]\\{5\\}$/']).expect(0, ['xx00', 'xx01'], ['', + 'alpha\n# comment\n\nbeta\ngamma\nEND\ndelta\n\nepsilon\n'], ['0', '47']) + reset() + run_in(['/^[[:alpha:]]\\{2,\\}$/']).expect(0, ['xx00', 'xx01'], ['', + 'alpha\n# comment\n\nbeta\ngamma\nEND\ndelta\n\nepsilon\n'], ['0', '47']) + reset() + // "beta" is the first line of exactly four letters, so the cut is after three. + run_in(['/^[[:alpha:]]\\{2,4\\}$/']).expect(0, ['xx00', 'xx01'], + ['alpha\n# comment\n\n', 'beta\ngamma\nEND\ndelta\n\nepsilon\n'], ['17', '30']) + reset() + // ^ is an anchor at the front of a branch and the character anywhere else. + run_in(['/a^b/']).expect_says(1, 'csplit: ‘/a^b/’: match not found') +} + +fn test_posix_classes_and_word_boundaries() { + write_input(words) + // The space inside "# comment" is what this finds, at offset 1 of that line. + run_in(['/[[:space:]]/']).expect(0, ['xx00', 'xx01'], + ['alpha\n', '# comment\n\nbeta\ngamma\nEND\ndelta\n\nepsilon\n'], ['6', '41']) + reset() + // \ needs a whole word, and neither "epsilon" nor "beta" gives it one. + run_in(['/\\/']).expect_says(1, 'csplit: ‘/\\\\/’: match not found') + reset() + // \w and \s are classes, so this matches the first line; \d and \a are only the + // letters they look like, so this wants a line holding nothing but a d. + run_in(['/^\\w\\+$/']).expect(0, ['xx00', 'xx01'], ['', + 'alpha\n# comment\n\nbeta\ngamma\nEND\ndelta\n\nepsilon\n'], ['0', '47']) + reset() + run_in(['/^d$/']).expect_says(1, 'csplit: ‘/^d$/’: match not found') + reset() + run_in(['/^$/']).expect(0, ['xx00', 'xx01'], ['alpha\n# comment\n', + '\nbeta\ngamma\nEND\ndelta\n\nepsilon\n'], ['16', '31']) +} + +// A last line without a newline is still a line, and the pieces are cut on the exact +// bytes rather than on lines put back together with newlines. +fn test_input_without_a_trailing_newline() { + write_input('1\n2\n3') + run_in(['/2/']).expect(0, ['xx00', 'xx01'], ['1\n', '2\n3'], ['2', '3']) + reset() + run_in(['2']).expect(0, ['xx00', 'xx01'], ['1\n', '2\n3'], ['2', '3']) +} + +fn test_empty_input() { + write_input('') + // A pattern that searches reports no match and prints nothing for the piece. + run_in(['/2/']).expect_says(1, 'csplit: ‘/2/’: match not found') + reset() + // A line number has a complaint of its own, and still leaves an empty first piece + // behind without reporting a size for it. + r := run_in(['1']) + r.expect_says(1, 'csplit: input disappeared') + assert r.stdout == '', 'printed [${r.stdout}]' + assert r.names == ['xx00'], '${r.names}' + assert r.contents == [''], '${r.contents}' +} + +fn test_input_that_cannot_be_opened() { + reset() + run(['no-such-file', '/3/'], '').expect_says(1, + "csplit: cannot open 'no-such-file' for reading: No such file or directory") +} + +// An operand of "-" reads standard input, which os.new_process can supply from a file. +fn test_reading_standard_input() { + reset() + os.write_file(stdin_name, ten) or { panic(err) } + run(['-', '/3/'], stdin_name).expect(0, ['xx00', 'xx01'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) + reset() + os.write_file(stdin_name, '1\n2\n3') or { panic(err) } + run(['-', '/2/'], stdin_name).expect(0, ['xx00', 'xx01'], ['1\n', '2\n3'], ['2', '3']) + reset() + os.write_file(stdin_name, ten) or { panic(err) } + run(['-', '/99/'], stdin_name).expect_says(1, 'csplit: ‘/99/’: match not found') +} + +fn test_options_after_the_operands() { + write_input(ten) + run_in(['/5/', '-s']).expect(0, ['xx00', 'xx01'], ['1\n2\n3\n4\n', '5\n6\n7\n8\n9\n10\n'], []string{}) + reset() + run_in(['--', '/3/']).expect(0, ['xx00', 'xx01'], ['1\n2\n', '3\n4\n5\n6\n7\n8\n9\n10\n'], [ + '4', + '17', + ]) + reset() + run_in(['-sz', '/5/']).expect(0, ['xx00', 'xx01'], ['1\n2\n3\n4\n', '5\n6\n7\n8\n9\n10\n'], []string{}) +} + +fn test_nothing_to_split_on() { + write_input(ten) + r := run([input_name], '') + assert r.code == 1 + assert r.stderr.starts_with('csplit: missing operand after ‘in.txt’'), r.stderr +} diff --git a/src/csplit/ere.v b/src/csplit/ere.v new file mode 100644 index 00000000..c44b37df --- /dev/null +++ b/src/csplit/ere.v @@ -0,0 +1,277 @@ +module main + +// ere.v matches the pattern language GNU csplit hands to /REGEXP/ and %REGEXP%. +// +// That language is a GNU *basic* regular expression, not the POSIX extended one +// the manual's word "REGEXP" suggests. Every claim below was measured against +// csplit 9.4 with a one-line input, where a match shows up as an empty first +// output file: +// +// ^a+$ matches "a+" but not "aa" so + is an ordinary character +// ^a\?$ matches "a" so \? is the optional operator +// ^ab{1}$ matches "ab{1}" so { is an ordinary character +// ^a|b$ matches "a|b" so | is an ordinary character +// ^\(a\)$ matches "a" so \( and \) are the grouping forms +// ^a^b$ matches "a^b" so ^ anchors only at the very start +// ^\101$ is refused as a back reference +// +// The zero-width \< \> \b \B and the shorthands \w \W \s \S are BRE features too, +// and are present, while \d \a \n \t are just the letters they look like. +// +// V's own `regex` module is not a substitute: its | binds to single tokens instead +// of to sequences, and it has neither POSIX character classes nor \b, so even a +// two-branch alternation cannot be expressed there at all. +// +// One deliberate difference: GNU matches a multibyte character as one character, so +// `.` covers `é` whole. This matcher works in bytes, so `.` covers one byte of it. +// That only shows up on input that is not ASCII. + +const ere_unmatched_open = 'Unmatched ( or \\(' +const ere_unmatched_close = 'Unmatched ) or \\)' +const ere_unmatched_brace = 'Unmatched \\{' +const ere_trailing_backslash = 'Trailing backslash' +const ere_bad_class_name = 'Invalid character class name' +const ere_bad_back_reference = 'Invalid back reference' +const ere_unmatched_bracket = 'Unmatched [ or [^' + +enum EreKind { + empty + literal + any + class + at_begin + at_end + word_begin + word_end + not_word_begin + seq + alt + repeat +} + +// EreSpan is one inclusive byte range of a bracket expression. +struct EreSpan { + min u8 + max u8 +} + +// EreNode is one thing to match. Children are held by reference so that the tree is +// shared rather than copied as the matcher walks it. It is marked @[heap] because a +// reference to a node outlives the call that received it. +@[heap] +struct EreNode { +mut: + kind EreKind + ch u8 + negated bool + spans []EreSpan + // A repeat body may match nothing, in which case there is no upper bound worth + // trying and unbounded says so. + min int + max int + unbounded bool + kids []&EreNode +} + +// EreCont is one entry of the continuation a match works through: a node still to +// match, or a repetition that may still take another copy of its body. Carrying the +// repetition on the continuation is what lets it remember how many copies it has +// taken without the node itself being mutable. +enum EreContKind { + node + repeat +} + +struct EreCont { +mut: + kind EreContKind + ref &EreNode + count int +} + +// Ere is a compiled pattern, ready to apply to many lines. +pub struct Ere { +mut: + root EreNode +} + +// compile parses a BRE pattern. The error carries GNU's own wording for the +// failure so that csplit can report it without restating it. +fn (mut e Ere) compile(pattern []u8) ! { + mut p := EreParser{ + pattern: pattern + } + e.root = p.parse_alt() or { return err } + // A parse that stopped early left a \) with nothing in front of it, which is the + // one thing a branch cannot end on: "^\)" is refused rather than read as a ^. + if p.pos < pattern.len { + return error(ere_unmatched_close) + } +} + +// matches reports whether the pattern matches anywhere in the line. csplit only ever +// asks that question, never where the match fell, so no position comes back. +// +// Every offset in the line is a possible start, which is why "12" has to be found by +// a pattern of "2" starting one byte in. +fn (e &Ere) matches(line []u8) bool { + for start in 0 .. line.len + 1 { + if e.run([EreCont{ + ref: &e.root + }], line, start) { + return true + } + } + return false +} + +// run consumes the continuation until it runs out. An empty continuation means the +// whole pattern matched, and whatever is left of the line does not matter. +fn (e &Ere) run(next []EreCont, line []u8, at int) bool { + if next.len == 0 { + return true + } + c := next[0] + rest := next[1..] + if c.kind == .repeat { + return e.repeat_from(c.ref, c.count, rest, line, at) + } + return e.step(c.ref, rest, line, at) +} + +// repeat_from tries one more copy of the body before settling for the copies it +// already has, which is what makes * and + greedy. A count past the length of the +// line bounds the loop, which is what stops a body that matches the empty string from +// repeating for ever. +fn (e &Ere) repeat_from(n &EreNode, count int, rest []EreCont, line []u8, at int) bool { + if (n.unbounded || count < n.max) && count <= line.len { + mut cont := [ + EreCont{ + ref: n.kids[0] + }, + EreCont{ + kind: .repeat + ref: n + count: count + 1 + }, + ] + cont << rest + if e.run(cont, line, at) { + return true + } + } + if count >= n.min { + return e.run(rest, line, at) + } + return false +} + +// step matches one node. +fn (e &Ere) step(n &EreNode, next []EreCont, line []u8, at int) bool { + match n.kind { + .empty { + return e.run(next, line, at) + } + .literal { + if at < line.len && line[at] == n.ch { + return e.run(next, line, at + 1) + } + return false + } + .any { + if at < line.len { + return e.run(next, line, at + 1) + } + return false + } + .class { + if at < line.len && e.class_matches(n, line[at]) { + return e.run(next, line, at + 1) + } + return false + } + .at_begin { + return at == 0 && e.run(next, line, at) + } + .at_end { + return at == line.len && e.run(next, line, at) + } + .word_begin { + return at_word_begin(line, at) && e.run(next, line, at) + } + .word_end { + return at_word_end(line, at) && e.run(next, line, at) + } + .not_word_begin { + return !at_word_begin(line, at) && e.run(next, line, at) + } + .seq { + mut cont := []EreCont{cap: n.kids.len + next.len} + for kid in n.kids { + cont << EreCont{ + ref: kid + } + } + cont << next + return e.run(cont, line, at) + } + .alt { + for kid in n.kids { + mut cont := [EreCont{ + ref: kid + }] + cont << next + if e.run(cont, line, at) { + return true + } + } + return false + } + .repeat { + mut cont := [ + EreCont{ + kind: .repeat + ref: n + }, + ] + cont << next + return e.run(cont, line, at) + } + } +} + +// at_word_begin reports whether at sits where \< matches: after the start of the line +// or a non-word byte, and on a word byte. +fn at_word_begin(line []u8, at int) bool { + if at >= line.len || !is_word_byte(line[at]) { + return false + } + return at == 0 || !is_word_byte(line[at - 1]) +} + +// at_word_end reports whether at sits where \> matches: on a word byte followed by +// the end of the line or a non-word byte. +fn at_word_end(line []u8, at int) bool { + if at >= line.len || !is_word_byte(line[at]) { + return false + } + return at + 1 == line.len || !is_word_byte(line[at + 1]) +} + +// class_matches tests one byte against a bracket expression, honouring the negation +// a leading ^ puts on it. +fn (e &Ere) class_matches(n &EreNode, b u8) bool { + mut hit := false + for s in n.spans { + if b >= s.min && b <= s.max { + hit = true + break + } + } + return hit != n.negated +} + +fn is_word_byte(b u8) bool { + return (b >= `a` && b <= `z`) || (b >= `A` && b <= `Z`) || (b >= `0` && b <= `9`) + || b == `_` +} diff --git a/src/csplit/ere_parser.v b/src/csplit/ere_parser.v new file mode 100644 index 00000000..e6415821 --- /dev/null +++ b/src/csplit/ere_parser.v @@ -0,0 +1,523 @@ +module main + +// ere_parser.v turns BRE source into the node tree that ere.v matches against. The +// grammar it accepts is the one measured in ere.v's header: +// +// alt := concat ('\|' concat)* +// concat := piece* +// piece := atom quantifier* +// quantifier := '*' | '\+' | '\?' | '\{' n [',' [m]] '\}' +// atom := '\(' alt '\)' | bracket | '.' | '^' | '$' | '\' escape | character +// +// Two rules are easy to get wrong and are both measured rather than assumed. A * with +// nothing in front of it to repeat is an ordinary character, which is what makes a * +// at the start of a pattern, or straight after \( or \|, a literal. And ^ and $ are +// anchors only at the very start or very end of the whole pattern, so ^a^b$ matches +// the text a^b. + +struct EreParser { +mut: + pattern []u8 + pos int + // at_start is true where a * would have nothing to repeat: through the first + // piece of a pattern, and straight after every \( and \|. + at_start bool = true + // repeatable is true where something a repetition could attach to has just been + // read. An anchor is not one, which is why ^\{ is a brace and not an interval. + repeatable bool +} + +struct EreInterval { +mut: + min int + max int +} + +fn ere_span(lo u8, hi u8) EreSpan { + return EreSpan{ + min: lo + max: hi + } +} + +fn ere_literal(b u8) &EreNode { + return &EreNode{ + kind: .literal + ch: b + } +} + +fn ere_marker(kind EreKind) &EreNode { + return &EreNode{ + kind: kind + } +} + +// posix_spans returns the byte ranges of a [:name:] class, or none when the name is +// not one. GNU refuses an unknown name rather than taking it literally, which is what +// the caller turns into an error. +fn posix_spans(name string) ?[]EreSpan { + return match name { + 'alnum' { + [ere_span(`0`, `9`), ere_span(`A`, `Z`), ere_span(`a`, `z`)] + } + 'alpha' { + [ere_span(`A`, `Z`), ere_span(`a`, `z`)] + } + 'blank' { + [ere_span(`\t`, `\t`), ere_span(` `, ` `)] + } + 'cntrl' { + [ere_span(0, 31), ere_span(127, 127)] + } + 'digit' { + [ere_span(`0`, `9`)] + } + 'graph' { + [ere_span(33, 126)] + } + 'lower' { + [ere_span(`a`, `z`)] + } + 'print' { + [ere_span(32, 126)] + } + 'punct' { + [ + ere_span(`!`, `/`), + ere_span(`:`, `@`), + ere_span(`[`, u8(96)), + ere_span(`{`, `~`), + ] + } + 'space' { + [ere_span(9, 13), ere_span(` `, ` `)] + } + 'upper' { + [ere_span(`A`, `Z`)] + } + 'xdigit' { + [ere_span(`0`, `9`), ere_span(`A`, `F`), ere_span(`a`, `f`)] + } + else { + none + } + } +} + +fn (p EreParser) at_end() bool { + return p.pos >= p.pattern.len +} + +fn (p EreParser) peek_is(want u8) bool { + return !p.at_end() && p.pattern[p.pos] == want +} + +fn (p EreParser) next_is(want u8) bool { + return p.pos + 1 < p.pattern.len && p.pattern[p.pos + 1] == want +} + +fn (mut p EreParser) take() u8 { + b := p.pattern[p.pos] + p.pos++ + return b +} + +fn (p EreParser) byte_at(i int) u8 { + return if i < p.pattern.len { p.pattern[i] } else { 0 } +} + +// at_alt_bar reports whether the parser is sitting on the \| that joins two branches. +// The unescaped | is an ordinary character in a BRE, so only this spelling counts. +fn (p EreParser) at_alt_bar() bool { + return p.peek_is(`\\`) && p.next_is(`|`) +} + +// at_close_group reports whether the parser is on the \) that ends the group it is +// inside, which parse_concat has to stop for so that the group can close. +fn (p EreParser) at_close_group() bool { + return p.peek_is(`\\`) && p.next_is(`)`) +} + +// parse_alt reads a concatenation, or several joined by \|. +fn (mut p EreParser) parse_alt() !&EreNode { + mut branches := [p.parse_concat() or { return err }] + for p.at_alt_bar() { + p.pos += 2 + p.at_start = true + p.repeatable = false + branches << p.parse_concat() or { return err } + } + if branches.len == 1 { + return branches[0] + } + return &EreNode{ + kind: .alt + kids: branches + } +} + +// parse_concat reads a run of pieces, stopping before whatever belongs to an +// enclosing group or alternation rather than to this branch. +fn (mut p EreParser) parse_concat() !&EreNode { + mut kids := []&EreNode{} + for !(p.at_end() || p.at_alt_bar() || p.at_close_group()) { + kids << p.parse_piece() or { return err } + } + return &EreNode{ + kind: .seq + kids: kids + } +} + +// parse_piece reads one atom with any quantifiers applied to it. +fn (mut p EreParser) parse_piece() !&EreNode { + mut n := p.parse_atom() or { return err } + for !p.at_end() { + q := p.parse_quantifier(n) or { return err } + if q.kind == .empty { + break + } + n = q + } + return n +} + +// parse_quantifier reads one repetition suffix applied to body, or reports an .empty +// node when the parser is not sitting on one, which is how parse_piece knows to stop. +fn (mut p EreParser) parse_quantifier(body &EreNode) !&EreNode { + mut lo := 0 + mut hi := 0 + if p.peek_is(`*`) { + // A literal when there is nothing in front of it to repeat. + if p.at_start { + return ere_marker(.empty) + } + p.pos++ + lo = 0 + hi = -1 + } else if p.peek_is(`\\`) && p.pos + 1 < p.pattern.len { + match p.pattern[p.pos + 1] { + `+` { + p.pos += 2 + lo = 1 + hi = -1 + } + `?` { + p.pos += 2 + lo = 0 + hi = 1 + } + `{` { + // A \{ opens an interval only when something a repetition could attach + // to came before it. Measured: /a\{b/ and /1\{2/ are refused as an + // unmatched \{, while /\{/ and /^\{/ are the character itself. + if !p.repeatable { + return ere_marker(.empty) + } + iv := p.parse_interval() or { return err } + lo = iv.min + hi = iv.max + } + else { + return ere_marker(.empty) + } + } + } else { + return ere_marker(.empty) + } + p.at_start = false + p.repeatable = true + return &EreNode{ + kind: .repeat + kids: [body] + min: lo + max: hi + unbounded: hi < 0 + } +} + +// parse_interval reads the body of a \{n\}, \{n,\} or \{n,m\} repetition. A { that does +// not open a well-formed interval is an ordinary character, which is measured rather +// than assumed: ^\{$ matches a line reading {, and \{2 matches nothing at all. +fn (mut p EreParser) parse_interval() !EreInterval { + mut i := p.pos + 2 + mut lo := 0 + mut digits := 0 + for p.byte_at(i) >= `0` && p.byte_at(i) <= `9` && i < p.pattern.len { + lo = lo * 10 + int(p.byte_at(i) - `0`) + digits++ + i++ + } + if digits == 0 { + return error(ere_unmatched_brace) + } + mut hi := lo + if p.byte_at(i) == `,` { + i++ + // \{n,} leaves the count open above, and the closing brace may be escaped. + if p.at_interval_end(i) { + hi = -1 + p.pos = i + if p.byte_at(i) == `}` { 1 } else { 2 } + return EreInterval{ + min: lo + max: hi + } + } + hi = 0 + digits = 0 + for p.byte_at(i) >= `0` && p.byte_at(i) <= `9` && i < p.pattern.len { + hi = hi * 10 + int(p.byte_at(i) - `0`) + digits++ + i++ + } + if digits == 0 || !p.at_interval_end(i) { + return error(ere_unmatched_brace) + } + p.pos = i + if p.byte_at(i) == `}` { 1 } else { 2 } + return EreInterval{ + min: lo + max: hi + } + } + if !p.at_interval_end(i) { + return error(ere_unmatched_brace) + } + // An unescaped closing brace ends the interval too, so a\{2\} and a\{2} mean + // the same thing. + p.pos = i + if p.byte_at(i) == `}` { 1 } else { 2 } + return EreInterval{ + min: lo + max: hi + } +} + +fn (p EreParser) at_interval_end(i int) bool { + return p.byte_at(i) == `}` || (p.byte_at(i) == `\\` && p.byte_at(i + 1) == `}`) +} + +// parse_atom reads one thing to match. +fn (mut p EreParser) parse_atom() !&EreNode { + if p.at_end() { + return error(ere_unmatched_open) + } + b := p.pattern[p.pos] + match b { + `\\` { + return p.parse_escape() + } + `[` { + return p.parse_bracket() + } + `.` { + p.pos++ + p.at_start = false + p.repeatable = true + return ere_marker(.any) + } + `^` { + // An anchor at the front of a branch, which is the front of the whole + // pattern and also the front of every piece after a \|. Measured two ways: + // "^a^b$" matches the text a^b, so a ^ anywhere else is the character, and + // "^e\|^a" matches "alpha", so the ^ after the bar still anchors. + mut anchor := p.at_start + p.pos++ + p.at_start = false + p.repeatable = false + if anchor { + return ere_marker(.at_begin) + } + return ere_literal(b) + } + `$` { + mut anchor := p.pos == p.pattern.len - 1 + p.pos++ + p.at_start = false + p.repeatable = false + if anchor { + return ere_marker(.at_end) + } + return ere_literal(b) + } + else { + // ( and ) are ordinary characters in a BRE: ^(a)$ matches the text (a) + // and not a. + p.pos++ + p.at_start = false + p.repeatable = true + return ere_literal(b) + } + } +} + +// parse_escape reads a backslash and whatever follows it. The operators are a short +// list; a backslash in front of anything else is dropped and the character stands for +// itself, which is why \d matches a d and \a matches an a. +fn (mut p EreParser) parse_escape() !&EreNode { + p.pos++ // the backslash + if p.at_end() { + return error(ere_trailing_backslash) + } + b := p.take() + match b { + `(` { + return p.parse_group() + } + `)` { + return error(ere_unmatched_close) + } + `<`, `b` { + p.at_start = false + p.repeatable = false + return ere_marker(.word_begin) + } + `>` { + p.at_start = false + p.repeatable = false + return ere_marker(.word_end) + } + `B` { + p.at_start = false + p.repeatable = false + return ere_marker(.not_word_begin) + } + `w` { + p.at_start = false + p.repeatable = true + return ere_shorthand(false) + } + `W` { + p.at_start = false + p.repeatable = true + return ere_shorthand(true) + } + `s` { + p.at_start = false + p.repeatable = true + return ere_space_class(false) + } + `S` { + p.at_start = false + p.repeatable = true + return ere_space_class(true) + } + `0`...`9` { + // A back reference. csplit refuses them, so this does too, with the same + // wording. + return error(ere_bad_back_reference) + } + else { + p.at_start = false + p.repeatable = true + return ere_literal(b) + } + } +} + +// parse_group reads the body of a \( ... \) group. +fn (mut p EreParser) parse_group() !&EreNode { + saved := p.at_start + p.at_start = true + p.repeatable = false + inner := p.parse_alt() or { return err } + p.at_start = saved + p.repeatable = true + if !p.at_close_group() { + return error(ere_unmatched_open) + } + p.pos += 2 + return inner +} + +// ere_shorthand builds the class behind \w and \W. +fn ere_shorthand(negated bool) &EreNode { + return &EreNode{ + kind: .class + negated: negated + spans: [ere_span(`0`, `9`), ere_span(`A`, `Z`), ere_span(`a`, `z`), ere_span(`_`, `_`)] + } +} + +// ere_space_class builds the class behind \s and \S. +fn ere_space_class(negated bool) &EreNode { + return &EreNode{ + kind: .class + negated: negated + spans: [ere_span(9, 13), ere_span(` `, ` `)] + } +} + +// parse_bracket reads a [ ... ] bracket expression, expanding any [:name:] classes +// into plain byte ranges so that matching needs no name lookup. +fn (mut p EreParser) parse_bracket() !&EreNode { + p.pos++ // the [ + mut n := &EreNode{ + kind: .class + } + if p.peek_is(`^`) { + n.negated = true + p.pos++ + } + // A ] straight after the [ or the [^ is the character itself. + mut first := true + for !p.at_end() { + if p.peek_is(`]`) && !first { + p.pos++ + p.at_start = false + // A bracket expression is something a repetition can attach to, which is + // what makes "[[:lower:]]\{4\}" four of them rather than a brace. + p.repeatable = true + return n + } + first = false + if end := p.find_class_name_end() { + n.spans << posix_spans(p.pattern[p.pos + 2..end].bytestr()) or { + return error(ere_bad_class_name) + } + p.pos = end + 2 + continue + } + lo := p.parse_bracket_char() or { return err } + // A range needs a - that is not the last character before the ]. + if p.peek_is(`-`) && !p.next_is(`]`) { + p.pos++ + hi := p.parse_bracket_char() or { return err } + n.spans << ere_span(lo, hi) + } else { + n.spans << ere_span(lo, lo) + } + } + return error(ere_unmatched_bracket) +} + +// find_class_name_end returns the index of the :] closing a [: :] class name, or none +// when the [: here is a literal [ followed by a colon. +fn (p EreParser) find_class_name_end() ?int { + if !p.peek_is(`[`) || !p.next_is(`:`) { + return none + } + mut i := p.pos + 2 + for i + 1 < p.pattern.len { + if p.pattern[i] == `:` && p.pattern[i + 1] == `]` { + return i + } + i++ + } + return none +} + +// parse_bracket_char reads one character inside a bracket expression, where a +// backslash is only there to make the next character literal. +fn (mut p EreParser) parse_bracket_char() !u8 { + if p.at_end() { + return error(ere_unmatched_bracket) + } + if !p.peek_is(`\\`) { + return p.take() + } + p.pos++ + if p.at_end() { + return error(ere_trailing_backslash) + } + return p.take() +}