diff options
Diffstat (limited to 'src/commands/uniq.js')
| -rw-r--r-- | src/commands/uniq.js | 279 |
1 files changed, 279 insertions, 0 deletions
diff --git a/src/commands/uniq.js b/src/commands/uniq.js new file mode 100644 index 0000000..90be7ab --- /dev/null +++ b/src/commands/uniq.js @@ -0,0 +1,279 @@ +#!/usr/bin/env bun + +import { closeSync, openSync } from "node:fs"; +import { createFdRecordReader, decodeSurrogateEscapedBytes, isWriteError, localeQuotedDiagnostic, localeQuotedEscapedDiagnostic, nodeErrorMessage, normalizeLongOptionByPrefix, normalizeLongOptionsByPrefix, parseOptions, pathDisplayName, readAll, shellEscapeLsName, systemErrorMessage, textInputDiagnosticName } from "../shared/common.js"; +import { UsageError, encodeSurrogateEscapedString, stderr, stdout } from "../shared/diagnostics.js"; +import { writeAll } from "../shared/text.js"; +import { defineCommand, runAsMain } from "../shared/command.js"; + +export const UNIQ_LONG_OPTIONS = ["count", "repeated", "all-repeated", "skip-fields", "group", "ignore-case", "skip-chars", "unique", "zero-terminated", "check-chars", "help", "version"]; + +export function uniqMetaOption(args) { + const longValueOptions = new Set(["skip-fields", "skip-chars", "check-chars"]); + const longOptionalValueOptions = new Set(["all-repeated", "group"]); + const shortValueOptions = new Set(["f", "s", "w"]); + const shortKnownOptions = new Set(["c", "d", "D", "f", "i", "s", "u", "w", "z"]); + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--") return null; + if (arg.startsWith("--")) { + const option = normalizeUniqLongOption(arg); + const [name, inlineValue] = option.slice(2).split(/=(.*)/s, 2); + if (!UNIQ_LONG_OPTIONS.includes(name)) return null; + if (name === "all-repeated" && inlineValue != null && !["none", "prepend", "separate"].includes(inlineValue)) return null; + if (name === "group" && inlineValue != null && !["prepend", "append", "separate", "both"].includes(inlineValue)) return null; + if ((option === "--help" || option === "--version") && inlineValue == null) return option; + if (longValueOptions.has(name)) { + const value = inlineValue ?? args[i + 1]; + if (value !== undefined) validateUniqMetaOptionValue(name, value); + if (inlineValue == null) i++; + } + else if (inlineValue == null && longOptionalValueOptions.has(name)) continue; + continue; + } + if (!arg.startsWith("-") || arg === "-") continue; + if (/^-\d+$/.test(arg) || /^\+\d+$/.test(arg)) continue; + for (let j = 1; j < arg.length; j++) { + const ch = arg[j]; + if (!shortKnownOptions.has(ch)) return null; + if (shortValueOptions.has(ch)) { + const inlineValue = arg.slice(j + 1); + const value = inlineValue === "" ? args[i + 1] : inlineValue; + if (value !== undefined) validateUniqMetaOptionValue(ch, value); + if (inlineValue === "") i++; + break; + } + } + } + return null; +} + +export function validateUniqMetaOptionValue(option, value) { + if (option === "skip-fields" || option === "f") validateUniqNumericOption(value, "fields to skip"); + else if (option === "skip-chars" || option === "s") validateUniqNumericOption(value, "bytes to skip"); + else if (option === "check-chars" || option === "w") validateUniqNumericOption(value, "bytes to compare"); +} + +export async function uniq(args) { + args = normalizeUniqArgs(normalizeUniqLongOptions(args)); + const { opts, operands } = parseOptions(args, { + short: { c: false, d: false, D: false, f: "value", i: false, s: "value", u: false, w: "value", z: false }, + long: { count: false, repeated: false, "all-repeated": "optional-value", "skip-fields": "value", group: "optional-value", "ignore-case": false, "skip-chars": "value", unique: false, "check-chars": "value", "zero-terminated": false, help: false, version: false }, + }); + const count = opts.c || opts.count; + const repeated = opts.d || opts.repeated; + const unique = opts.u || opts.unique; + const allRepeated = opts.D || opts["all-repeated"]; + const group = opts.group; + if (allRepeated && count) throw new UsageError("printing all duplicated lines and repeat counts is meaningless", true); + if (opts["all-repeated"] != null && opts["all-repeated"] !== true) validateUniqMethod("all-repeated", opts["all-repeated"], ["none", "prepend", "separate"]); + if (opts.group != null && opts.group !== true) validateUniqMethod("group", opts.group, ["prepend", "append", "separate", "both"]); + if (group && (count || repeated || allRepeated || unique)) throw new UsageError("--group is mutually exclusive with -c/-d/-D/-u", true); + validateUniqNumericOptions(opts); + const inputFile = operands[0] ?? "-"; + const outputFile = operands[1]; + if (operands.length > 2) throw new UsageError(`extra operand ${localeQuotedEscapedDiagnostic(operands[2])}`, true); + if (!outputFile && canStreamUniqFile(opts)) return streamUniqFile(inputFile, opts); + let inputBytes; + try { + inputBytes = await readAll(inputFile); + } catch (error) { + const inputName = textInputDiagnosticName(inputFile); + const message = error?.code === "EISDIR" ? `error reading ${shellEscapeLsName(pathDisplayName(inputFile), true)}: ${systemErrorMessage(error)}` : `${inputName}: ${systemErrorMessage(error)}`; + stderr(inputFile === "-" ? `uniq: ${nodeErrorMessage(error)}\n` : `uniq: ${message}\n`); + return 1; + } + const text = decodeSurrogateEscapedBytes(inputBytes); + const sep = opts.z || opts["zero-terminated"] ? "\0" : "\n"; + const lines = text.endsWith(sep) ? text.slice(0, -sep.length).split(sep) : text === "" ? [] : text.split(sep); + if (repeated && unique) { + if (outputFile) await writeAll(outputFile, ""); + return 0; + } + const rows = []; + for (const line of lines) { + const key = uniqKey(line, opts); + const last = rows.at(-1); + if (last?.key === key) { + last.count++; + last.lines.push(line); + } else { + rows.push({ key, line, lines: [line], count: 1 }); + } + } + const outLines = []; + const allRepeatedMethod = opts["all-repeated"] === true || opts.D ? "none" : opts["all-repeated"]; + const groupMethod = opts.group === true ? "separate" : opts.group; + for (const row of rows) { + if (opts.group) { + pushUniqGroup(outLines, row.lines, groupMethod, opts, row.count, ["separate", "prepend", "append", "both"]); + } else if (allRepeated) { + if (row.count > 1) pushUniqGroup(outLines, row.lines, allRepeatedMethod, opts, row.count, ["none", "prepend", "separate"]); + } else if (repeated) { + if (row.count > 1) outLines.push(formatUniqLine(row.line, row.count, opts)); + } else if (unique) { + if (row.count === 1) outLines.push(formatUniqLine(row.line, row.count, opts)); + } else { + outLines.push(formatUniqLine(row.line, row.count, opts)); + } + } + const out = outLines.length ? outLines.join(sep) + sep : ""; + if (outputFile) { + try { + await writeAll(outputFile, encodeSurrogateEscapedString(out)); + } catch (error) { + stderr(`uniq: ${textInputDiagnosticName(outputFile)}: ${systemErrorMessage(error)}\n`); + return 1; + } + } else { + stdout(out); + } + return 0; +} + +export function canStreamUniqFile(opts) { + if (canStreamUniq(opts)) return true; + const allRepeatedMethod = opts["all-repeated"] === true || opts.D ? "none" : opts["all-repeated"]; + return allRepeatedMethod === "none" + && !(opts.c || opts.count || opts.d || opts.repeated || opts.u || opts.unique || opts.group); +} + +export function streamUniqFile(file, opts) { + const separator = opts.z || opts["zero-terminated"] ? 0 : 10; + const sep = Uint8Array.of(separator); + const allRepeated = Boolean(opts.D || opts["all-repeated"]); + let fd; + try { + fd = file === "-" ? 0 : openSync(file, "r"); + const reader = createFdRecordReader(fd, separator); + let current = null; + let count = 0; + while (true) { + const record = reader.next(); + if (record == null) break; + const line = decodeSurrogateEscapedBytes(record); + const key = uniqKey(line, opts); + if (current?.key === key) { + count++; + if (allRepeated) { + if (count === 2) { + stdout(current.record); + stdout(sep); + } + stdout(record); + stdout(sep); + } + } else { + current = { key, record: Buffer.from(record) }; + count = 1; + if (!allRepeated) { + stdout(record); + stdout(sep); + } + } + } + return 0; + } catch (error) { + if (isWriteError(error)) throw error; + const message = error?.code === "EISDIR" ? `error reading ${shellEscapeLsName(pathDisplayName(file), true)}: ${systemErrorMessage(error)}` : `${textInputDiagnosticName(file)}: ${systemErrorMessage(error)}`; + stderr(file === "-" ? `uniq: ${nodeErrorMessage(error)}\n` : `uniq: ${message}\n`); + return 1; + } finally { + if (fd != null && fd !== 0) closeSync(fd); + } +} + +export function canStreamUniq(opts) { + return !(opts.c || opts.count || opts.d || opts.repeated || opts.u || opts.unique || opts.D || opts["all-repeated"] || opts.group); +} + +export function normalizeUniqArgs(args) { + const out = []; + let scanning = true; + let valueNext = false; + for (const arg of args) { + if (!scanning) { + out.push(arg); + } else if (valueNext) { + out.push(arg); + valueNext = false; + } else if (arg === "-f" || arg === "--skip-fields" || arg === "-s" || arg === "--skip-chars" || arg === "-w" || arg === "--check-chars") { + out.push(arg); + valueNext = true; + } else if (/^-\d+$/.test(arg)) { + out.push("-f", arg.slice(1)); + } else if (/^\+\d+$/.test(arg)) { + out.push("-s", arg.slice(1)); + } else { + if (arg === "--") scanning = false; + else if (!arg.startsWith("-")) scanning = false; + out.push(arg); + } + } + return out; +} + +export function normalizeUniqLongOptions(args) { + return normalizeLongOptionsByPrefix(args, UNIQ_LONG_OPTIONS); +} + +export function normalizeUniqLongOption(arg) { + return normalizeLongOptionByPrefix(arg, UNIQ_LONG_OPTIONS); +} + +export function validateUniqNumericOptions(opts) { + validateUniqNumericOption(opts.f ?? opts["skip-fields"], "fields to skip"); + validateUniqNumericOption(opts.s ?? opts["skip-chars"], "bytes to skip"); + validateUniqNumericOption(opts.w ?? opts["check-chars"], "bytes to compare"); +} + +export function validateUniqNumericOption(value, description) { + if (value == null) return; + if (/^\+?\d+$/.test(String(value))) return; + throw new UsageError(`${value}: invalid number of ${description}`); +} + +export function validateUniqMethod(option, method, allowed) { + if (allowed.includes(method)) return; + const kind = method === "" ? "ambiguous" : "invalid"; + throw new UsageError(`${kind} argument ${localeQuotedEscapedDiagnostic(method)} for ${localeQuotedDiagnostic(`--${option}`)}\nValid arguments are:\n${allowed.map((value) => ` - ${localeQuotedDiagnostic(value)}`).join("\n")}`, true); +} + +export function pushUniqGroup(outLines, lines, method, opts, count, allowed) { + if (!allowed.includes(method)) throw new UsageError(`invalid group method: ${method}`); + if (method === "prepend" || (method === "both" && outLines.length === 0)) outLines.push(""); + else if (method === "separate" && outLines.length) outLines.push(""); + for (const line of lines) outLines.push(formatUniqLine(line, count, opts)); + if (method === "append" || method === "both") outLines.push(""); +} + +export function formatUniqLine(line, count, opts) { + return `${opts.c || opts.count ? String(count).padStart(7) + " " : ""}${line}`; +} + +export function uniqKey(line, opts) { + let key = line; + const skipFields = Number(opts.f ?? opts["skip-fields"] ?? 0); + const skipChars = Number(opts.s ?? opts["skip-chars"] ?? 0); + const width = opts.w ?? opts["check-chars"]; + if (!Number.isInteger(skipFields) || skipFields < 0) throw new UsageError(`invalid field skip: ${opts.f ?? opts["skip-fields"]}`); + if (!Number.isInteger(skipChars) || skipChars < 0) throw new UsageError(`invalid character skip: ${opts.s ?? opts["skip-chars"]}`); + let offset = 0; + for (let field = 0; field < skipFields && offset < key.length; field++) { + while (offset < key.length && /\s/u.test(key[offset])) offset++; + while (offset < key.length && !/\s/u.test(key[offset])) offset++; + } + key = [...key.slice(offset)].slice(skipChars).join(""); + if (width !== undefined) { + const max = Number(width); + if (!Number.isInteger(max) || max < 0) throw new UsageError(`invalid compare width: ${width}`); + key = [...key].slice(0, max).join(""); + } + if (opts.i || opts["ignore-case"]) key = key.toLowerCase(); + return key; +} + +const singleCall = defineCommand("uniq", uniq, uniqMetaOption); +export default singleCall; + +if (import.meta.main) await runAsMain(singleCall); |
