Complete Guide to Text Processing Tools: Sorting, Replacing, and Manipulation

// Sort lines alphabetically
const text = "banana\napple\ncherry\ndate";
const lines = text.split('\n');

// A-Z (ascending)
const sortedAsc = [...lines].sort();
console.log(sortedAsc.join('\n'));
// apple, banana, cherry, date

// Z-A (descending)
const sortedDesc = [...lines].sort().reverse();
console.log(sortedDesc.join('\n'));

// Case-insensitive sort
const sortedInsensitive = [...lines].sort((a, b) =>
  a.toLowerCase().localeCompare(b.toLowerCase())
);

// Numerical sort
const numbers = ["10", "2", "1", "20"];
const sortedNumbers = [...numbers].sort((a, b) =>
  parseInt(a) - parseInt(b)
);
console.log(sortedNumbers.join('\n'));
// 1, 2, 10, 20

// Natural sort (for file names like file1, file2, file10)
const naturalSort = [...lines].sort((a, b) =>
  a.localeCompare(b, undefined, { numeric: true, sensitivity: 'base' })
);

# Sort lines alphabetically
text = """banana
apple
cherry
date"""

lines = text.split('\n')

# A-Z (ascending)
sorted_asc = sorted(lines)
print('\n'.join(sorted_asc))

# Z-A (descending)
sorted_desc = sorted(lines, reverse=True)

# Case-insensitive sort
sorted_insensitive = sorted(lines, key=str.lower)

# Numerical sort
numbers = ["10", "2", "1", "20"]
sorted_numbers = sorted(numbers, key=int)
print('\n'.join(sorted_numbers))
# 1, 2, 10, 20

# Sort by length
sorted_by_length = sorted(lines, key=len)

# Custom sort (by second character)
sorted_custom = sorted(lines, key=lambda x: x[1] if len(x) > 1 else '')

# Natural sort (requires natsort library)
from natsort import natsorted
files = ["file1.txt", "file10.txt", "file2.txt"]
sorted_natural = natsorted(files)
print('\n'.join(sorted_natural))
# file1.txt, file2.txt, file10.txt

# Sort lines alphabetically
sort file.txt

# Sort in reverse order
sort -r file.txt

# Case-insensitive sort
sort -f file.txt

# Numerical sort
sort -n numbers.txt

# Sort by second column (tab-separated)
sort -k2 data.txt

# Sort and remove duplicates
sort -u file.txt

# Natural sort (version sort)
sort -V files.txt

# Sort by month
sort -M months.txt

# Sort in place (overwrite original file)
sort -o file.txt file.txt

# Complex example: sort by second field numerically
sort -t, -k2n data.csv

// Replace first occurrence
const text = "Hello world, hello universe";
const replaced = text.replace("hello", "hi");
console.log(replaced);
// "Hello world, hi universe"

// Replace all occurrences
const replacedAll = text.replace(/hello/g, "hi");
console.log(replacedAll);
// "Hello world, hi universe"

// Replace using replaceAll (ES2021)
const replacedAll2 = text.replaceAll("hello", "hi");

// Multiple replacements
let result = text;
const replacements = {
  "hello": "hi",
  "world": "earth",
  "universe": "cosmos"
};

Object.entries(replacements).forEach(([find, replace]) => {
  result = result.replace(new RegExp(find, 'gi'), replace);
});
console.log(result);
// "hi earth, hi cosmos"

// Replace digits with X
const text = "Order 123 shipped, tracking: 456789";
const masked = text.replace(/\d/g, 'X');
console.log(masked);
// "Order XXX shipped, tracking: XXXXXX"

// Replace email addresses with [REDACTED]
const withEmail = "Contact: [email protected] or [email protected]";
const redacted = withEmail.replace(
  /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
  '[REDACTED]'
);

// Replace URLs with links
const withUrls = "Visit https://example.com or http://test.org";
const linked = withUrls.replace(
  /(https?:\/\/[^\s]+)/g,
  '<a href="$1">$1</a>'
);

// Replace dates (MM/DD/YYYY to YYYY-MM-DD)
const usDate = "Meeting on 12/25/2024 and 01/15/2025";
const isoDate = usDate.replace(
  /(\d{2})\/(\d{2})\/(\d{4})/g,
  '$3-$1-$2'
);
console.log(isoDate);
// "Meeting on 2024-12-25 and 2025-01-15"

// Case-insensitive replacement
const text = "Hello HELLO hello HeLLo";

// Using regex with 'i' flag
const replaced = text.replace(/hello/gi, 'hi');
console.log(replaced);
// "hi hi hi hi"

// Python example
"""
text = "Hello HELLO hello HeLLo"
import re
replaced = re.sub(r'hello', 'hi', text, flags=re.IGNORECASE)
print(replaced)
# "hi hi hi hi"
"""

// Preserve original case
function replacePreserveCase(text, find, replace) {
  return text.replace(new RegExp(find, 'gi'), (match) => {
    if (match === match.toUpperCase()) return replace.toUpperCase();
    if (match === match.toLowerCase()) return replace.toLowerCase();
    if (match[0] === match[0].toUpperCase()) {
      return replace[0].toUpperCase() + replace.slice(1).toLowerCase();
    }
    return replace;
  });
}

const result = replacePreserveCase("Hello HELLO hello", "hello", "hi");
console.log(result);
// "Hi HI hi"

// Chain multiple replacements
function multiReplace(text, replacements) {
  return Object.entries(replacements).reduce(
    (result, [find, replace]) =>
      result.replace(new RegExp(find, 'g'), replace),
    text
  );
}

const text = "The quick brown fox jumps over the lazy dog";
const result = multiReplace(text, {
  'quick': 'fast',
  'brown': 'red',
  'lazy': 'sleepy',
  'dog': 'cat'
});
console.log(result);
// "The fast red fox jumps over the sleepy cat"

// Using a map for multiple replacements (Python)
"""
def multi_replace(text, replacements):
    for find, replace in replacements.items():
        text = text.replace(find, replace)
    return text

replacements = {
    'quick': 'fast',
    'brown': 'red',
    'lazy': 'sleepy'
}
result = multi_replace(text, replacements)
"""

// Extract and reformat phone numbers
const phone = "(555) 123-4567";
const formatted = phone.replace(
  /\((\d{3})\)\s(\d{3})-(\d{4})/,
  '$1-$2-$3'
);
console.log(formatted);
// "555-123-4567"

// Convert camelCase to snake_case
function camelToSnake(text) {
  return text.replace(/([a-z])([A-Z])/g, '$1_$2').toLowerCase();
}
console.log(camelToSnake('myVariableName'));
// "my_variable_name"

// Swap first and last name
const name = "Doe, John";
const swapped = name.replace(/(\w+),\s(\w+)/, '$2 $1');
console.log(swapped);
// "John Doe"

// Extract domain from email
const email = "[email protected]";
const domain = email.replace(/^[^@]+@(.+)$/, '$1');
console.log(domain);
// "example.com"

// Add line numbers
const text = "Line one\nLine two\nLine three";
let lineNum = 0;
const numbered = text.replace(/^/gm, () => `${++lineNum}. `);
console.log(numbered);
// "1. Line one\n2. Line two\n3. Line three"

// Remove duplicates preserving order
function removeDuplicates(text) {
  const lines = text.split('\n');
  const seen = new Set();
  const unique = [];

  for (const line of lines) {
    if (!seen.has(line)) {
      seen.add(line);
      unique.push(line);
    }
  }

  return unique.join('\n');
}

const text = "apple\nbanana\napple\ncherry\nbanana\ndate";
console.log(removeDuplicates(text));
// "apple\nbanana\ncherry\ndate"

// Python version
"""
def remove_duplicates(text):
    lines = text.split('\n')
    seen = set()
    unique = []

    for line in lines:
        if line not in seen:
            seen.add(line)
            unique.append(line)

    return '\n'.join(unique)
"""

// Using array methods (loses order)
const unique = [...new Set(text.split('\n'))].join('\n');

// Case-insensitive duplicate removal
function removeDuplicatesInsensitive(text) {
  const lines = text.split('\n');
  const seen = new Set();
  const unique = [];

  for (const line of lines) {
    const lower = line.toLowerCase();
    if (!seen.has(lower)) {
      seen.add(lower);
      unique.push(line);
    }
  }

  return unique.join('\n');
}

const text = "Apple\nBANANA\napple\nCherry\nbanana\nDATE";
console.log(removeDuplicatesInsensitive(text));
// "Apple\nBANANA\nCherry\nDATE"

// Python version
"""
def remove_duplicates_case_insensitive(text):
    lines = text.split('\n')
    seen = set()
    unique = []

    for line in lines:
        lower = line.lower()
        if lower not in seen:
            seen.add(lower)
            unique.append(line)

    return '\n'.join(unique)
"""

// Count duplicate occurrences
function countDuplicates(text) {
  const lines = text.split('\n');
  const counts = new Map();

  for (const line of lines) {
    counts.set(line, (counts.get(line) || 0) + 1);
  }

  // Return lines with counts > 1
  const duplicates = [];
  for (const [line, count] of counts.entries()) {
    if (count > 1) {
      duplicates.push(`${line} (appears ${count} times)`);
    }
  }

  return duplicates.join('\n');
}

const text = "apple\nbanana\napple\ncherry\nbanana\napple";
console.log(countDuplicates(text));
// "apple (appears 3 times)\nbanana (appears 2 times)"

// Get frequency of all lines
function getFrequency(text) {
  const lines = text.split('\n');
  const freq = {};

  for (const line of lines) {
    freq[line] = (freq[line] || 0) + 1;
  }

  return freq;
}

// Keep only lines that appear more than once
function keepOnlyDuplicates(text) {
  const lines = text.split('\n');
  const counts = new Map();

  // Count occurrences
  for (const line of lines) {
    counts.set(line, (counts.get(line) || 0) + 1);
  }

  // Keep lines with count > 1 (preserving duplicates)
  return lines.filter(line => counts.get(line) > 1).join('\n');
}

const text = "apple\nbanana\napple\ncherry\nbanana\napple\ndate";
console.log(keepOnlyDuplicates(text));
// "apple\nbanana\napple\nbanana\napple"

// Keep only unique duplicates (one occurrence each)
function uniqueDuplicates(text) {
  const lines = text.split('\n');
  const counts = new Map();

  for (const line of lines) {
    counts.set(line, (counts.get(line) || 0) + 1);
  }

  return [...counts.entries()]
    .filter(([_, count]) => count > 1)
    .map(([line]) => line)
    .join('\n');
}
console.log(uniqueDuplicates(text));
// "apple\nbanana"

// Remove all empty lines
function removeEmptyLines(text) {
  return text.split('\n')
    .filter(line => line.trim() !== '')
    .join('\n');
}

const text = "Line 1\n\nLine 2\n  \nLine 3";
console.log(removeEmptyLines(text));
// "Line 1\nLine 2\nLine 3"

// Using regex
const cleaned = text.replace(/^\s*\n/gm, '');

// Python version
"""
def remove_empty_lines(text):
    return '\n'.join(line for line in text.split('\n') if line.strip())

# Using regex
import re
cleaned = re.sub(r'^\s*\n', '', text, flags=re.MULTILINE)
"""

// Command-line (sed)
"""
sed '/^$/d' file.txt
"""

// Remove lines containing only whitespace
function removeWhitespaceLines(text) {
  return text.split('\n')
    .filter(line => line.trim().length > 0)
    .join('\n');
}

// Using regex
function removeWhitespaceLinesRegex(text) {
  return text.replace(/^\s*$\n/gm, '');
}

// Differentiate between empty and whitespace-only lines
function analyzeLines(text) {
  const lines = text.split('\n');
  return lines.map(line => ({
    content: line,
    isEmpty: line === '',
    isWhitespaceOnly: line.trim() === '' && line !== '',
    hasContent: line.trim().length > 0
  }));
}

const text = "Line 1\n\nLine 2\n   \n\t\nLine 3";
console.log(analyzeLines(text));

// Collapse multiple empty lines into one
function collapseEmptyLines(text) {
  return text.replace(/\n\s*\n\s*\n/g, '\n\n');
}

// More robust version
function collapseMultipleEmptyLines(text) {
  return text.replace(/\n{3,}/g, '\n\n');
}

const text = "Line 1\n\n\n\nLine 2\n\n\n\n\nLine 3";
console.log(collapseEmptyLines(text));
// "Line 1\n\nLine 2\n\nLine 3"

// Python version
"""
import re

def collapse_empty_lines(text):
    return re.sub(r'\n{3,}', '\n\n', text)
"""

// Command-line (sed)
"""
sed '/^$/N;/^\n$/D' file.txt
"""

// Trim leading and trailing whitespace from each line
function trimLines(text) {
  return text.split('\n')
    .map(line => line.trim())
    .join('\n');
}

// Trim only leading whitespace
function trimLeading(text) {
  return text.split('\n')
    .map(line => line.replace(/^\s+/, ''))
    .join('\n');
}

// Trim only trailing whitespace
function trimTrailing(text) {
  return text.split('\n')
    .map(line => line.replace(/\s+$/, ''))
    .join('\n');
}

const text = "  Line 1  \n\tLine 2\t\n   Line 3   ";
console.log(trimLines(text));
// "Line 1\nLine 2\nLine 3"

// Using regex to trim entire text
const trimmed = text.replace(/^\s+|\s+$/gm, '');

// Remove all whitespace characters
function removeAllWhitespace(text) {
  return text.replace(/\s+/g, '');
}

const text = "Hello   World\n\tTest  123";
console.log(removeAllWhitespace(text));
// "HelloWorldTest123"

// Remove specific whitespace types
function removeSpaces(text) {
  return text.replace(/ /g, '');
}

function removeTabs(text) {
  return text.replace(/\t/g, '');
}

function removeNewlines(text) {
  return text.replace(/\n/g, '');
}

// Python version
"""
def remove_all_whitespace(text):
    return ''.join(text.split())

# Or using regex
import re
cleaned = re.sub(r'\s+', '', text)
"""

// Normalize whitespace (multiple spaces to single space)
function normalizeWhitespace(text) {
  return text.replace(/\s+/g, ' ').trim();
}

const text = "Hello    World\n\tTest   123";
console.log(normalizeWhitespace(text));
// "Hello World Test 123"

// Normalize within lines but preserve line breaks
function normalizePreservingLines(text) {
  return text.split('\n')
    .map(line => line.replace(/\s+/g, ' ').trim())
    .join('\n');
}

// Python version
"""
import re

def normalize_whitespace(text):
    return re.sub(r'\s+', ' ', text).strip()
"""

// Command-line (tr)
"""
tr -s ' ' < file.txt
"""

// Convert tabs to spaces (4 spaces per tab)
function tabsToSpaces(text, spaces = 4) {
  return text.replace(/\t/g, ' '.repeat(spaces));
}

// Convert tabs to spaces (2 spaces per tab)
const twoSpaces = text => text.replace(/\t/g, '  ');

// Convert spaces to tabs (4 spaces = 1 tab)
function spacesToTabs(text, spaces = 4) {
  const pattern = new RegExp(' '.repeat(spaces), 'g');
  return text.replace(pattern, '\t');
}

const text = "function test() {\n\treturn 42;\n}";
console.log(tabsToSpaces(text));
// "function test() {\n    return 42;\n}"

// Python version
"""
def tabs_to_spaces(text, spaces=4):
    return text.replace('\t', ' ' * spaces)
"""

// Command-line (expand)
"""
expand -t 4 file.txt
unexpand -t 4 file.txt
"""

// Trim leading and trailing whitespace
const text = "  Hello World  \n\t";

// Standard trim
console.log(text.trim());
// "Hello World"

// Trim start only
console.log(text.trimStart()); // or trimLeft()
// "Hello World  \n\t"

// Trim end only
console.log(text.trimEnd()); // or trimRight()
// "  Hello World"

// Custom trim function with specific characters
function trimChars(text, chars) {
  const pattern = new RegExp(`^[${chars}]+|[${chars}]+$`, 'g');
  return text.replace(pattern, '');
}

console.log(trimChars("...Hello...", "."));
// "Hello"

// Python version
"""
text = "  Hello World  \n\t"
print(text.strip())        # "Hello World"
print(text.lstrip())       # "Hello World  \n\t"
print(text.rstrip())       # "  Hello World"
"""

// Repeat text N times
function repeatText(text, count) {
  return text.repeat(count);
}

console.log(repeatText("Hello ", 3));
// "Hello Hello Hello "

// Repeat with Array methods
const repeated = Array(5).fill("*").join("");
console.log(repeated);
// "*****"

// Repeat each line N times
function repeatLines(text, count) {
  return text.split('\n')
    .flatMap(line => Array(count).fill(line))
    .join('\n');
}

const lines = "Line 1\nLine 2";
console.log(repeatLines(lines, 2));
// "Line 1\nLine 1\nLine 2\nLine 2"

// Python version
"""
def repeat_text(text, count):
    return text * count

print("Hello " * 3)
# "Hello Hello Hello "
"""

// Repeat with separator
function repeatWithSeparator(text, count, separator = ", ") {
  return Array(count).fill(text).join(separator);
}

console.log(repeatWithSeparator("item", 5, ", "));
// "item, item, item, item, item"

// Create lists
console.log(repeatWithSeparator("- Item", 3, "\n"));
// "- Item\n- Item\n- Item"

// Generate CSV rows
function generateCSVRows(data, count) {
  return Array(count).fill(data.join(",")).join("\n");
}

const row = ["Name", "Age", "City"];
console.log(generateCSVRows(row, 3));

// Python version
"""
def repeat_with_separator(text, count, separator=", "):
    return separator.join([text] * count)
"""

// Repeat with sequential numbers
function repeatNumbered(text, count, start = 1) {
  return Array.from({ length: count }, (_, i) =>
    `${start + i}. ${text}`
  ).join('\n');
}

console.log(repeatNumbered("Item", 5));
// "1. Item\n2. Item\n3. Item\n4. Item\n5. Item"

// Custom numbering format
function repeatCustomNumbered(text, count, format = (n) => `[${n}]`) {
  return Array.from({ length: count }, (_, i) =>
    `${format(i + 1)} ${text}`
  ).join('\n');
}

console.log(repeatCustomNumbered("Task", 3, n => `Task#${n.toString().padStart(3, '0')}`));
// "Task#001 Task\nTask#002 Task\nTask#003 Task"

// Generate test data with IDs
function generateTestData(template, count) {
  return Array.from({ length: count }, (_, i) =>
    template.replace(/{id}/g, i + 1)
      .replace(/{index}/g, i)
  ).join('\n');
}

console.log(generateTestData('{"id": {id}, "name": "User {id}"}', 3));

// Generate test data
function generateTestData(count) {
  const templates = [
    name => `User${name}`,
    email => `user${email}@test.com`,
    id => `USR-${id.toString().padStart(5, '0')}`
  ];

  return Array.from({ length: count }, (_, i) => ({
    id: templates[2](i + 1),
    name: templates[0](i + 1),
    email: templates[1](i + 1)
  }));
}

console.log(JSON.stringify(generateTestData(3), null, 2));

// Generate repeated pattern for testing
function generatePattern(pattern, count) {
  const chars = pattern.split('');
  let result = '';
  for (let i = 0; i < count; i++) {
    result += chars[i % chars.length];
  }
  return result;
}

console.log(generatePattern('ABC', 10));
// "ABCABCABCA"

// Generate lorem ipsum alternative
function generatePlaceholder(words = 50) {
  const placeholder = ["lorem", "ipsum", "dolor", "sit", "amet"];
  return Array.from({ length: words }, (_, i) =>
    placeholder[i % placeholder.length]
  ).join(' ');
}

// Common regex patterns for text processing

// Email validation
const emailPattern = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/;

// Phone number (US format)
const phonePattern = /^\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}$/;

// URL
const urlPattern = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/;

// IP address (IPv4)
const ipPattern = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;

// Date (YYYY-MM-DD)
const datePattern = /^\d{4}-\d{2}-\d{2}$/;

// Credit card number
const creditCardPattern = /^(?:\d{4}[\s-]?){3}\d{4}$/;

// Hexadecimal color
const colorPattern = /^#?([a-fA-F0-9]{6}|[a-fA-F0-9]{3})$/;

// Username (alphanumeric, underscore, 3-16 chars)
const usernamePattern = /^[a-zA-Z0-9_]{3,16}$/;

// Strong password (min 8 chars, uppercase, lowercase, number, special)
const passwordPattern = /^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[@$!%*?&])[A-Za-z\d@$!%*?&]{8,}$/;

// Whitespace
const whitespacePattern = /\s+/g;

// Word boundary
const wordPattern = /\b\w+\b/g;

// Extract email addresses
function extractEmails(text) {
  const pattern = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
  return text.match(pattern) || [];
}

const text = "Contact [email protected] or [email protected] for info";
console.log(extractEmails(text));
// ["[email protected]", "[email protected]"]

// Extract URLs
function extractUrls(text) {
  const pattern = /https?:\/\/[^\s]+/g;
  return text.match(pattern) || [];
}

// Extract phone numbers
function extractPhones(text) {
  const pattern = /\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}/g;
  return text.match(pattern) || [];
}

// Extract hashtags
function extractHashtags(text) {
  const pattern = /#\w+/g;
  return text.match(pattern) || [];
}

// Extract numbers
function extractNumbers(text) {
  const pattern = /-?\d+\.?\d*/g;
  return text.match(pattern).map(Number);
}

// Extract words
function extractWords(text) {
  const pattern = /\b[a-zA-Z]+\b/g;
  return text.match(pattern) || [];
}

// Validation functions using regex

function isValidEmail(email) {
  const pattern = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/;
  return pattern.test(email);
}

function isValidUrl(url) {
  const pattern = /^https?:\/\/[^\s]+$/;
  return pattern.test(url);
}

function isValidIPv4(ip) {
  const pattern = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
  return pattern.test(ip);
}

function isValidHexColor(color) {
  const pattern = /^#?([a-fA-F0-9]{6}|[a-fA-F0-9]{3})$/;
  return pattern.test(color);
}

function isStrongPassword(password) {
  const pattern = /^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[@$!%*?&])[A-Za-z\d@$!%*?&]{8,}$/;
  return pattern.test(password);
}

// Comprehensive validation with error messages
function validateInput(value, type) {
  const validators = {
    email: {
      pattern: /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/,
      message: "Invalid email format"
    },
    phone: {
      pattern: /^\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}$/,
      message: "Invalid phone number format"
    },
    url: {
      pattern: /^https?:\/\/[^\s]+$/,
      message: "Invalid URL format"
    }
  };

  const validator = validators[type];
  if (!validator) return { valid: false, error: "Unknown validation type" };

  return validator.pattern.test(value)
    ? { valid: true }
    : { valid: false, error: validator.message };
}

# Basic substitution
sed 's/old/new/' file.txt

# Global replacement (all occurrences)
sed 's/old/new/g' file.txt

# Case-insensitive replacement
sed 's/old/new/gi' file.txt

# Delete lines containing pattern
sed '/pattern/d' file.txt

# Delete empty lines
sed '/^$/d' file.txt

# Print only matching lines
sed -n '/pattern/p' file.txt

# Replace on specific line number
sed '5s/old/new/' file.txt

# Replace in range of lines
sed '10,20s/old/new/g' file.txt

# Multiple operations
sed -e 's/old1/new1/g' -e 's/old2/new2/g' file.txt

# In-place editing (modify file)
sed -i 's/old/new/g' file.txt

# Backup before in-place edit
sed -i.bak 's/old/new/g' file.txt

# Delete lines 5-10
sed '5,10d' file.txt

# Add text before/after pattern
sed '/pattern/i\New line before' file.txt
sed '/pattern/a\New line after' file.txt

# Print specific column (space-separated)
awk '{print $1}' file.txt

# Print multiple columns
awk '{print $1, $3}' file.txt

# Print with custom separator
awk -F, '{print $1, $2}' file.csv

# Print lines longer than 80 characters
awk 'length > 80' file.txt

# Sum values in column
awk '{sum += $1} END {print sum}' numbers.txt

# Print line numbers
awk '{print NR, $0}' file.txt

# Filter by condition
awk '$3 > 100 {print $0}' data.txt

# Count lines
awk 'END {print NR}' file.txt

# Calculate average
awk '{sum += $1; count++} END {print sum/count}' numbers.txt

# Print unique lines (like uniq)
awk '!seen[$0]++' file.txt

# Format output
awk '{printf "%-10s %5d\n", $1, $2}' file.txt

# Multiple conditions
awk '$1 == "active" && $2 > 50 {print $0}' data.txt

# Sort and remove duplicates
sort file.txt | uniq

# Sort and remove duplicates (shortcut)
sort -u file.txt

# Count duplicate occurrences
sort file.txt | uniq -c

# Show only duplicates
sort file.txt | uniq -d

# Show only unique lines (no duplicates)
sort file.txt | uniq -u

# Case-insensitive unique
sort -f file.txt | uniq -i

# Count and sort by frequency
sort file.txt | uniq -c | sort -rn

# Remove adjacent duplicates (no sorting)
uniq file.txt

# Unique by field
sort -k2 file.txt | uniq -f 1

# Convert lowercase to uppercase
tr 'a-z' 'A-Z' < file.txt

# Convert uppercase to lowercase
tr 'A-Z' 'a-z' < file.txt

# Delete specific characters
tr -d '0-9' < file.txt

# Delete whitespace
tr -d ' \t' < file.txt

# Squeeze repeated characters
tr -s ' ' < file.txt

# Replace characters
tr ',' '\n' < file.csv

# Delete all non-printable characters
tr -cd '[:print:]' < file.txt

# Convert spaces to tabs
tr ' ' '\t' < file.txt

# Convert tabs to spaces
tr '\t' ' ' < file.txt

# Remove all newlines
tr -d '\n' < file.txt

# ROT13 encoding
tr 'A-Za-z' 'N-ZA-Mn-za-m' < file.txt

# Sort, remove duplicates, and count
cat file.txt | sort | uniq -c | sort -rn

# Extract emails from file
grep -oE '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' file.txt | sort -u

# Find most common words
cat file.txt | tr -cs 'A-Za-z' '\n' | tr 'A-Z' 'a-z' | sort | uniq -c | sort -rn | head -20

# Remove duplicates and empty lines
cat file.txt | grep -v '^$' | sort -u

# Count total words
cat file.txt | wc -w

# Find lines with phone numbers
grep -E '\(?[0-9]{3}\)?[-. ]?[0-9]{3}[-. ]?[0-9]{4}' file.txt

# Replace and remove empty lines
sed 's/old/new/g' file.txt | sed '/^$/d'

# Extract URLs and download
grep -oP 'https?://[^\s]+' file.txt | xargs wget

# CSV processing: extract column 2, sort, unique count
cut -d, -f2 file.csv | sort | uniq -c

# Log analysis: count unique IPs
awk '{print $1}' access.log | sort | uniq -c | sort -rn

// Process large files efficiently

// ❌ Bad: Load entire file into memory
function processFileBad(text) {
  return text.split('\n')
    .map(line => line.trim())
    .filter(line => line)
    .join('\n');
}

// ✅ Good: Stream processing for large files (Node.js)
const fs = require('fs');
const readline = require('readline');

async function processFileStreaming(inputPath, outputPath) {
  const fileStream = fs.createReadStream(inputPath);
  const output = fs.createWriteStream(outputPath);

  const rl = readline.createInterface({
    input: fileStream,
    crlfDelay: Infinity
  });

  for await (const line of rl) {
    const processed = line.trim();
    if (processed) {
      output.write(processed + '\n');
    }
  }

  output.end();
}

// ❌ Bad: Multiple passes over data
function multiplePassesBad(text) {
  let result = text.replace(/old1/g, 'new1');
  result = result.replace(/old2/g, 'new2');
  result = result.replace(/old3/g, 'new3');
  return result;
}

// ✅ Good: Single pass with combined regex
function singlePassGood(text) {
  return text.replace(/old1|old2|old3/g, match => {
    const replacements = { old1: 'new1', old2: 'new2', old3: 'new3' };
    return replacements[match];
  });
}

// Robust error handling for text processing

function safeTextProcess(text, operation) {
  try {
    // Validate input
    if (typeof text !== 'string') {
      throw new TypeError('Input must be a string');
    }

    if (text.length === 0) {
      return { success: true, result: '', warning: 'Empty input' };
    }

    // Perform operation
    const result = operation(text);

    return { success: true, result };

  } catch (error) {
    return {
      success: false,
      error: error.message,
      stack: error.stack
    };
  }
}

// Handle encoding issues
function safeReadFile(filepath, encoding = 'utf8') {
  const fs = require('fs');

  try {
    return fs.readFileSync(filepath, encoding);
  } catch (error) {
    if (error.code === 'ENOENT') {
      throw new Error(`File not found: ${filepath}`);
    }
    if (error.code === 'EACCES') {
      throw new Error(`Permission denied: ${filepath}`);
    }
    // Try different encoding
    try {
      return fs.readFileSync(filepath, 'latin1');
    } catch (e) {
      throw new Error(`Unable to read file: ${error.message}`);
    }
  }
}

// Validate regex patterns
function safeRegexReplace(text, pattern, replacement) {
  try {
    const regex = new RegExp(pattern, 'g');
    return text.replace(regex, replacement);
  } catch (error) {
    throw new Error(`Invalid regex pattern: ${error.message}`);
  }
}