Complete Guide to URL Parsing and Manipulation

https://user:[email protected]:8080/path/to/resource?key=value&foo=bar#section

└─┬─┘   └──┬───┘ └────┬──────┘└┬─┘└───────┬────────┘└────────┬──────────┘└───┬───┘
scheme  userinfo     host    port      path              query          fragment

Full breakdown:
• protocol:   https
• username:   user
• password:   pass
• hostname:   example.com
• port:       8080
• pathname:   /path/to/resource
• search:     ?key=value&foo=bar
• hash:       #section

// Modern JavaScript URL API
const url = new URL('https://api.example.com:8080/users?page=2&limit=10#results');

console.log(url.protocol);    // 'https:'
console.log(url.hostname);    // 'api.example.com'
console.log(url.port);        // '8080'
console.log(url.pathname);    // '/users'
console.log(url.search);      // '?page=2&limit=10'
console.log(url.hash);        // '#results'

// Parse query parameters
const params = new URLSearchParams(url.search);
console.log(params.get('page'));   // '2'
console.log(params.get('limit'));  // '10'

// Get all components
console.log({
  href: url.href,
  origin: url.origin,           // 'https://api.example.com:8080'
  host: url.host,               // 'api.example.com:8080'
  username: url.username,
  password: url.password
});

from urllib.parse import urlparse, parse_qs

url = 'https://api.example.com:8080/users?page=2&limit=10#results'
parsed = urlparse(url)

print(f"Scheme: {parsed.scheme}")      # https
print(f"Hostname: {parsed.hostname}")  # api.example.com
print(f"Port: {parsed.port}")          # 8080
print(f"Path: {parsed.path}")          # /users
print(f"Query: {parsed.query}")        # page=2&limit=10
print(f"Fragment: {parsed.fragment}")  # results

# Parse query parameters
params = parse_qs(parsed.query)
print(params)  # {'page': ['2'], 'limit': ['10']}
print(params['page'][0])  # '2'

# Reconstruct URL
from urllib.parse import urlunparse
reconstructed = urlunparse(parsed)
print(reconstructed)  # Original URL

// Node.js URL module (legacy)
const url = require('url');

const myUrl = 'https://api.example.com:8080/users?page=2&limit=10#results';
const parsed = url.parse(myUrl, true); // true parses query string

console.log({
  protocol: parsed.protocol,  // 'https:'
  host: parsed.host,          // 'api.example.com:8080'
  hostname: parsed.hostname,  // 'api.example.com'
  port: parsed.port,          // '8080'
  pathname: parsed.pathname,  // '/users'
  search: parsed.search,      // '?page=2&limit=10'
  query: parsed.query,        // { page: '2', limit: '10' }
  hash: parsed.hash           // '#results'
});

// Modern Node.js (use WHATWG URL API - same as browser)
const { URL } = require('url');
const myUrl2 = new URL('https://api.example.com:8080/users?page=2&limit=10');
console.log(myUrl2.searchParams.get('page')); // '2'

<?php
$url = 'https://user:[email protected]:8080/users?page=2&limit=10#results';
$parsed = parse_url($url);

print_r($parsed);
/*
Array (
    [scheme] => https
    [host] => api.example.com
    [port] => 8080
    [user] => user
    [pass] => pass
    [path] => /users
    [query] => page=2&limit=10
    [fragment] => results
)
*/

// Parse query string
parse_str($parsed['query'], $params);
print_r($params);
// Array ( [page] => 2 [limit] => 10 )

// Get specific component
$scheme = parse_url($url, PHP_URL_SCHEME);   // https
$host = parse_url($url, PHP_URL_HOST);       // api.example.com
$port = parse_url($url, PHP_URL_PORT);       // 8080
?>

// JavaScript - Build query string
const params = {
  search: 'laptop',
  category: 'electronics',
  minPrice: 500,
  maxPrice: 1500,
  sort: 'price_asc'
};

// Method 1: URLSearchParams
const searchParams = new URLSearchParams(params);
console.log(searchParams.toString());
// search=laptop&category=electronics&minPrice=500&maxPrice=1500&sort=price_asc

// Method 2: Manual building
const queryString = Object.entries(params)
  .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`)
  .join('&');

// Full URL
const baseUrl = 'https://api.example.com/products';
const fullUrl = `${baseUrl}?${searchParams}`;
console.log(fullUrl);

// Array parameters
const tags = ['javascript', 'typescript', 'node.js'];
tags.forEach(tag => searchParams.append('tags', tag));
// tags=javascript&tags=typescript&tags=node.js

// Python equivalent
/*
from urllib.parse import urlencode

params = {
    'search': 'laptop',
    'category': 'electronics',
    'minPrice': 500
}
query_string = urlencode(params)
# search=laptop&category=electronics&minPrice=500
*/

// Parse query string from URL
const url = 'https://example.com/search?q=javascript&tags=tutorial&tags=beginner&page=1';

// Method 1: URLSearchParams
const params = new URLSearchParams(new URL(url).search);

// Get single value
console.log(params.get('q'));        // 'javascript'
console.log(params.get('page'));     // '1'

// Get all values for a key
console.log(params.getAll('tags'));  // ['tutorial', 'beginner']

// Check if parameter exists
console.log(params.has('q'));        // true
console.log(params.has('limit'));    // false

// Iterate over all parameters
for (const [key, value] of params) {
  console.log(`${key}: ${value}`);
}

// Convert to object
const paramsObj = Object.fromEntries(params);
console.log(paramsObj);
// { q: 'javascript', tags: 'beginner', page: '1' }
// Note: Only keeps last value for duplicate keys

// Get all parameters including duplicates
const allParams = {};
for (const [key, value] of params) {
  if (allParams[key]) {
    if (Array.isArray(allParams[key])) {
      allParams[key].push(value);
    } else {
      allParams[key] = [allParams[key], value];
    }
  } else {
    allParams[key] = value;
  }
}
console.log(allParams);
// { q: 'javascript', tags: ['tutorial', 'beginner'], page: '1' }

// Update URL query parameters
function updateQueryParams(url, updates) {
  const urlObj = new URL(url);
  const params = urlObj.searchParams;

  // Update parameters
  Object.entries(updates).forEach(([key, value]) => {
    if (value === null || value === undefined) {
      params.delete(key);  // Remove parameter
    } else if (Array.isArray(value)) {
      params.delete(key);  // Clear existing
      value.forEach(v => params.append(key, v));  // Add all
    } else {
      params.set(key, value);  // Set/update
    }
  });

  return urlObj.toString();
}

// Usage
const original = 'https://example.com/search?q=javascript&page=1';

// Update page
const updated1 = updateQueryParams(original, { page: 2 });
// https://example.com/search?q=javascript&page=2

// Add new parameter
const updated2 = updateQueryParams(original, { sort: 'recent', limit: 10 });
// https://example.com/search?q=javascript&page=1&sort=recent&limit=10

// Remove parameter
const updated3 = updateQueryParams(original, { page: null });
// https://example.com/search?q=javascript

// Update multiple tags
const updated4 = updateQueryParams(original, {
  tags: ['tutorial', 'beginner', 'javascript']
});
// https://example.com/search?q=javascript&page=1&tags=tutorial&tags=beginner&tags=javascript

Character  | URL Encoded | Description
-----------|-------------|----------------------------------
(space)    | %20 or +    | Space character
!          | %21         | Exclamation mark
"          | %22         | Double quote
#          | %23         | Hash/fragment identifier
$          | %24         | Dollar sign
%          | %25         | Percent (must be encoded!)
&          | %26         | Ampersand (query separator)
'          | %27         | Single quote
(          | %28         | Left parenthesis
)          | %29         | Right parenthesis
*          | %2A         | Asterisk
+          | %2B         | Plus sign
,          | %2C         | Comma
/          | %2F         | Forward slash
:          | %3A         | Colon
;          | %3B         | Semicolon
=          | %3D         | Equals (key-value separator)
?          | %3F         | Question mark (query start)
@          | %40         | At sign
[          | %5B         | Left square bracket
\          | %5C         | Backslash
]          | %5D         | Right square bracket
{          | %7B         | Left curly brace
}          | %7D         | Right curly brace
|          | %7C         | Pipe

Safe (never encoded): A-Z a-z 0-9 - _ . ~

// JavaScript URL encoding

// encodeURI - Encode entire URL (preserves :, /, ?, &, =)
const url = 'https://example.com/search?q=hello world&tags=web dev';
console.log(encodeURI(url));
// https://example.com/search?q=hello%20world&tags=web%20dev

// encodeURIComponent - Encode URL component (encodes everything except A-Z a-z 0-9 - _ . ! ~ * ' ( ))
const searchTerm = 'hello world & special chars: @#$%';
console.log(encodeURIComponent(searchTerm));
// hello%20world%20%26%20special%20chars%3A%20%40%23%24%25

// Building URL with parameters
const baseUrl = 'https://api.example.com/search';
const params = {
  q: 'javascript tutorial',
  author: 'John Doe',
  tags: 'web & mobile'
};

const queryString = Object.entries(params)
  .map(([key, value]) =>
    `${encodeURIComponent(key)}=${encodeURIComponent(value)}`
  )
  .join('&');

const fullUrl = `${baseUrl}?${queryString}`;
console.log(fullUrl);
// https://api.example.com/search?q=javascript%20tutorial&author=John%20Doe&tags=web%20%26%20mobile

// Decoding
console.log(decodeURIComponent('hello%20world%20%26%20special'));
// hello world & special

// Python equivalent
/*
from urllib.parse import quote, unquote, urlencode

# Encode component
encoded = quote('hello world & special')
# hello%20world%20%26%20special

# Decode
decoded = unquote(encoded)
# hello world & special

# Build query string
params = {'q': 'javascript tutorial', 'tags': 'web & mobile'}
query = urlencode(params)
# q=javascript+tutorial&tags=web+%26+mobile
*/

const text = 'Hello World! @#$%^&*()';

// encodeURI - Preserves URL structure characters
console.log(encodeURI(text));
// Hello%20World!%20@#$%25%5E&*()
// Preserves: @ # & * ( )

// encodeURIComponent - Encodes everything except unreserved chars
console.log(encodeURIComponent(text));
// Hello%20World!%20%40%23%24%25%5E%26*()
// Encodes: @ # & but preserves * ( )

// When to use each:
// encodeURI - Full URLs
const fullUrl = 'https://example.com/path?q=hello world';
console.log(encodeURI(fullUrl));
// https://example.com/path?q=hello%20world (preserves structure)

// encodeURIComponent - URL parameters/components
const baseUrl = 'https://example.com/search';
const query = 'hello world';
const url = `${baseUrl}?q=${encodeURIComponent(query)}`;
// https://example.com/search?q=hello%20world

// Common mistake: Using wrong encoding
const wrongUrl = `${baseUrl}?url=${encodeURI('https://other.com?param=value')}`;
// ❌ Wrong: https://example.com/search?url=https://other.com?param=value
// Creates invalid URL with multiple ?

const rightUrl = `${baseUrl}?url=${encodeURIComponent('https://other.com?param=value')}`;
// ✅ Right: https://example.com/search?url=https%3A%2F%2Fother.com%3Fparam%3Dvalue

// Encode JSON for URL transmission

// Method 1: Standard URL encoding
const jsonData = {
  filters: {
    category: 'electronics',
    priceRange: { min: 100, max: 500 }
  },
  sort: 'price_asc',
  page: 1
};

const jsonString = JSON.stringify(jsonData);
const encoded = encodeURIComponent(jsonString);
console.log(encoded);
// %7B%22filters%22%3A%7B%22category%22%3A%22electronics%22...

// Build URL
const url = `https://example.com/search?config=${encoded}`;
console.log(url);

// Decode
function decodeJsonFromUrl(url) {
  const params = new URLSearchParams(new URL(url).search);
  const encodedJson = params.get('config');
  if (!encodedJson) return null;

  try {
    return JSON.parse(decodeURIComponent(encodedJson));
  } catch (error) {
    console.error('Invalid JSON in URL:', error);
    return null;
  }
}

const decoded = decodeJsonFromUrl(url);
console.log(decoded);
// { filters: { category: 'electronics', priceRange: { min: 100, max: 500 }}, ... }

// Method 2: LZ-String compression (for large JSON)
// npm install lz-string
const LZString = require('lz-string');

function encodeJsonForUrl(data) {
  const json = JSON.stringify(data);
  return LZString.compressToEncodedURIComponent(json);
}

function decodeJsonFromUrl(compressed) {
  const json = LZString.decompressFromEncodedURIComponent(compressed);
  return JSON.parse(json);
}

const compressed = encodeJsonForUrl(jsonData);
console.log(`Original: ${encoded.length} chars`);
console.log(`Compressed: ${compressed.length} chars`);
// Compression is significant for large objects

// Base64 encoding for JSON in URLs

function jsonToBase64Url(obj) {
  const json = JSON.stringify(obj);
  // Use btoa for browser, Buffer for Node.js
  const base64 = typeof window !== 'undefined'
    ? btoa(json)
    : Buffer.from(json).toString('base64');

  // Make URL-safe: replace +, /, = with URL-safe characters
  return base64
    .replace(/\+/g, '-')
    .replace(/\//g, '_')
    .replace(/=/g, '');  // Remove padding
}

function base64UrlToJson(base64Url) {
  // Restore standard Base64
  let base64 = base64Url
    .replace(/-/g, '+')
    .replace(/_/g, '/');

  // Restore padding
  while (base64.length % 4) {
    base64 += '=';
  }

  // Decode
  const json = typeof window !== 'undefined'
    ? atob(base64)
    : Buffer.from(base64, 'base64').toString('utf-8');

  return JSON.parse(json);
}

// Usage
const data = {
  user: { id: 123, name: 'John Doe' },
  preferences: { theme: 'dark', lang: 'en' }
};

const encoded = jsonToBase64Url(data);
console.log(encoded);
// eyJ1c2VyIjp7ImlkIjoxMjMsIm5hbWUiOiJKb2huIERvZSJ9LCJwcmVmZXJlbmNlcyI6eyJ0aGVtZSI6ImRhcmsiLCJsYW5nIjoiZW4ifX0

const url = `https://example.com/app?state=${encoded}`;
console.log(url);

const decoded = base64UrlToJson(encoded);
console.log(decoded);
// { user: { id: 123, name: 'John Doe' }, preferences: { theme: 'dark', lang: 'en' }}

// Python equivalent
/*
import json
import base64

def json_to_base64_url(obj):
    json_str = json.dumps(obj)
    base64_bytes = base64.b64encode(json_str.encode('utf-8'))
    base64_str = base64_bytes.decode('utf-8')
    # Make URL-safe
    return base64_str.replace('+', '-').replace('/', '_').rstrip('=')

def base64_url_to_json(base64_url):
    # Restore standard Base64
    base64_str = base64_url.replace('-', '+').replace('_', '/')
    # Restore padding
    base64_str += '=' * (4 - len(base64_str) % 4)
    json_str = base64.b64decode(base64_str).decode('utf-8')
    return json.loads(json_str)
*/

// Real-world JSON URL examples

// 1. Shareable search filters
const searchState = {
  query: 'laptop',
  filters: {
    brand: ['Dell', 'HP'],
    priceMin: 500,
    priceMax: 1500,
    rating: 4
  },
  sort: 'price_asc',
  page: 1
};

const shareableUrl = `https://shop.example.com/search?state=${
  encodeURIComponent(JSON.stringify(searchState))
}`;

// 2. Chart/visualization configuration
const chartConfig = {
  type: 'line',
  data: {
    labels: ['Jan', 'Feb', 'Mar'],
    datasets: [{ label: 'Sales', data: [100, 200, 150] }]
  },
  options: { responsive: true }
};

const chartUrl = `https://charts.example.com/?config=${
  jsonToBase64Url(chartConfig)
}`;

// 3. Map view state
const mapState = {
  center: { lat: 40.7128, lng: -74.0060 },
  zoom: 12,
  markers: [
    { id: 1, lat: 40.7580, lng: -73.9855, label: 'Times Square' },
    { id: 2, lat: 40.7484, lng: -73.9857, label: 'Empire State' }
  ]
};

const mapUrl = `https://maps.example.com/?view=${
  jsonToBase64Url(mapState)
}`;

// 4. Form pre-fill
const formData = {
  name: 'John Doe',
  email: '[email protected]',
  country: 'US',
  interests: ['javascript', 'web-dev']
};

const formUrl = `https://example.com/register?prefill=${
  jsonToBase64Url(formData)
}`;

// 5. A/B test variant configuration
const experimentConfig = {
  experimentId: 'homepage-test-2024',
  variant: 'B',
  features: {
    newLayout: true,
    showBanner: false,
    ctaColor: 'blue'
  }
};

const testUrl = `https://example.com/?exp=${
  jsonToBase64Url(experimentConfig)
}`;

// Encode XML for URL transmission

function encodeXmlForUrl(xmlString) {
  // Method 1: Standard URL encoding
  return encodeURIComponent(xmlString);
}

function encodeXmlForUrlBase64(xmlString) {
  // Method 2: Base64 URL encoding (better for large XML)
  const base64 = typeof window !== 'undefined'
    ? btoa(xmlString)
    : Buffer.from(xmlString).toString('base64');

  return base64
    .replace(/\+/g, '-')
    .replace(/\//g, '_')
    .replace(/=/g, '');
}

// Example XML
const xmlData = `<?xml version="1.0" encoding="UTF-8"?>
<order>
  <id>12345</id>
  <customer name="John Doe" email="[email protected]"/>
  <items>
    <item id="1" quantity="2">Laptop</item>
    <item id="2" quantity="1">Mouse</item>
  </items>
  <total currency="USD">1299.99</total>
</order>`;

// Encode with URL encoding
const urlEncoded = encodeXmlForUrl(xmlData);
console.log(`URL Encoded length: ${urlEncoded.length}`);

// Encode with Base64
const base64Encoded = encodeXmlForUrlBase64(xmlData);
console.log(`Base64 Encoded length: ${base64Encoded.length}`);

// Build URL
const url1 = `https://api.example.com/process?xml=${urlEncoded}`;
const url2 = `https://api.example.com/process?xml=${base64Encoded}&encoding=base64`;

console.log('URL with standard encoding:', url1);
console.log('URL with Base64:', url2);

// Python equivalent
/*
from urllib.parse import quote
import base64

xml_data = """<?xml version="1.0"?>
<order>
  <id>12345</id>
  <customer name="John Doe"/>
</order>"""

# URL encoding
url_encoded = quote(xml_data)

# Base64 encoding
base64_encoded = base64.b64encode(xml_data.encode()).decode()
base64_url_safe = base64_encoded.replace('+', '-').replace('/', '_').rstrip('=')
*/

// Decode XML from URL

function decodeXmlFromUrl(encodedXml, encoding = 'url') {
  if (encoding === 'base64') {
    // Decode Base64
    let base64 = encodedXml
      .replace(/-/g, '+')
      .replace(/_/g, '/');

    // Restore padding
    while (base64.length % 4) {
      base64 += '=';
    }

    return typeof window !== 'undefined'
      ? atob(base64)
      : Buffer.from(base64, 'base64').toString('utf-8');
  } else {
    // Decode URL encoding
    return decodeURIComponent(encodedXml);
  }
}

// Usage
const url = 'https://api.example.com/process?xml=PD94bWwgdmVyc2lvbj0iMS4wIj8-PG9yZGVyPjxpZD4xMjM0NTwvaWQ-PC9vcmRlcj4&encoding=base64';

const urlObj = new URL(url);
const encodedXml = urlObj.searchParams.get('xml');
const encoding = urlObj.searchParams.get('encoding') || 'url';

const decodedXml = decodeXmlFromUrl(encodedXml, encoding);
console.log(decodedXml);

// Parse XML (if needed)
function parseXml(xmlString) {
  if (typeof window !== 'undefined') {
    // Browser
    const parser = new DOMParser();
    return parser.parseFromString(xmlString, 'text/xml');
  } else {
    // Node.js
    const { parseStringPromise } = require('xml2js');
    return parseStringPromise(xmlString);
  }
}

// Extract data from XML
parseXml(decodedXml).then(parsed => {
  console.log('Parsed XML:', parsed);
});

// Best practices for XML in URLs

// 1. Minify XML before encoding
function minifyXml(xml) {
  return xml
    .replace(/\n/g, '')           // Remove newlines
    .replace(/\s{2,}/g, ' ')      // Replace multiple spaces
    .replace(/> </g, '><')         // Remove spaces between tags
    .trim();
}

const xml = `<?xml version="1.0"?>
<order>
  <id>12345</id>
  <total>99.99</total>
</order>`;

const minified = minifyXml(xml);
console.log('Original:', xml.length, 'chars');
console.log('Minified:', minified.length, 'chars');

// 2. Use Base64 for better compression
function xmlToBase64Url(xml) {
  const minified = minifyXml(xml);
  return jsonToBase64Url(minified);  // Reuse Base64 function
}

// 3. Consider XML to JSON conversion
function xmlToJson(xml) {
  // For simple XML, convert to JSON for better compression
  // Use libraries like xml2js or fast-xml-parser
  const { parseString } = require('xml2js');

  return new Promise((resolve, reject) => {
    parseString(xml, (err, result) => {
      if (err) reject(err);
      else resolve(result);
    });
  });
}

// 4. Use attribute-based XML for smaller size
const verboseXml = `
<order>
  <id>12345</id>
  <customer>John Doe</customer>
  <email>[email protected]</email>
</order>`;

const compactXml = `
<order id="12345" customer="John Doe" email="[email protected]" />`;

console.log('Verbose:', verboseXml.length);
console.log('Compact:', compactXml.length);

// 5. Validate decoded XML
function validateXml(xmlString) {
  try {
    if (typeof window !== 'undefined') {
      const parser = new DOMParser();
      const doc = parser.parseFromString(xmlString, 'text/xml');
      const parseError = doc.querySelector('parsererror');
      return !parseError;
    } else {
      // Node.js validation
      const { parseString } = require('xml2js');
      parseString(xmlString, (err) => {
        return !err;
      });
    }
  } catch (error) {
    return false;
  }
}

// URL Builder class for complex URL construction

class UrlBuilder {
  constructor(baseUrl) {
    this.url = new URL(baseUrl);
  }

  // Set path
  setPath(path) {
    this.url.pathname = path;
    return this;
  }

  // Add path segment
  addPath(segment) {
    const path = this.url.pathname.endsWith('/')
      ? this.url.pathname
      : this.url.pathname + '/';
    this.url.pathname = path + segment;
    return this;
  }

  // Set/add query parameter
  setParam(key, value) {
    if (Array.isArray(value)) {
      this.url.searchParams.delete(key);
      value.forEach(v => this.url.searchParams.append(key, v));
    } else if (value !== null && value !== undefined) {
      this.url.searchParams.set(key, value);
    }
    return this;
  }

  // Add multiple parameters
  setParams(params) {
    Object.entries(params).forEach(([key, value]) => {
      this.setParam(key, value);
    });
    return this;
  }

  // Remove parameter
  removeParam(key) {
    this.url.searchParams.delete(key);
    return this;
  }

  // Set hash
  setHash(hash) {
    this.url.hash = hash.startsWith('#') ? hash : `#${hash}`;
    return this;
  }

  // Set port
  setPort(port) {
    this.url.port = port;
    return this;
  }

  // Build final URL
  build() {
    return this.url.toString();
  }

  // Get URL object
  toUrl() {
    return new URL(this.url.toString());
  }
}

// Usage
const url = new UrlBuilder('https://api.example.com')
  .setPath('/v1/users')
  .setParams({
    page: 2,
    limit: 20,
    sort: 'name',
    filters: ['active', 'verified']
  })
  .setHash('results')
  .build();

console.log(url);
// https://api.example.com/v1/users?page=2&limit=20&sort=name&filters=active&filters=verified#results

// Dynamic URL building
const builder = new UrlBuilder('https://api.example.com')
  .setPath('/search');

if (searchQuery) {
  builder.setParam('q', searchQuery);
}

if (filters.category) {
  builder.setParam('category', filters.category);
}

if (sortOrder) {
  builder.setParam('sort', sortOrder);
}

const finalUrl = builder.build();

// Resolve relative URLs

// Browser/Node.js URL API handles relative URLs
const baseUrl = 'https://example.com/path/to/page.html';

// Relative paths
console.log(new URL('./other.html', baseUrl).href);
// https://example.com/path/to/other.html

console.log(new URL('../parent.html', baseUrl).href);
// https://example.com/path/parent.html

console.log(new URL('/absolute/path.html', baseUrl).href);
// https://example.com/absolute/path.html

// Query strings and hashes
console.log(new URL('?query=value', baseUrl).href);
// https://example.com/path/to/page.html?query=value

console.log(new URL('#section', baseUrl).href);
// https://example.com/path/to/page.html#section

// Full URLs (ignores base)
console.log(new URL('https://other.com/page', baseUrl).href);
// https://other.com/page

// Helper function
function resolveUrl(url, base) {
  try {
    return new URL(url, base).href;
  } catch (error) {
    console.error('Invalid URL:', error);
    return null;
  }
}

// Usage in web scraping
const pageUrl = 'https://example.com/blog/post-1';
const links = [
  './post-2',
  '../about',
  '/contact',
  'https://external.com'
];

const absoluteLinks = links.map(link => resolveUrl(link, pageUrl));
console.log(absoluteLinks);
// [
//   'https://example.com/blog/post-2',
//   'https://example.com/about',
//   'https://example.com/contact',
//   'https://external.com'
// ]

// Python equivalent
/*
from urllib.parse import urljoin

base_url = 'https://example.com/path/to/page.html'
relative_url = '../other.html'
absolute_url = urljoin(base_url, relative_url)
# https://example.com/path/other.html
*/

// URL normalization for comparison and deduplication

function normalizeUrl(url) {
  const urlObj = new URL(url);

  // 1. Convert to lowercase (scheme, host)
  urlObj.protocol = urlObj.protocol.toLowerCase();
  urlObj.hostname = urlObj.hostname.toLowerCase();

  // 2. Remove default ports
  if (
    (urlObj.protocol === 'http:' && urlObj.port === '80') ||
    (urlObj.protocol === 'https:' && urlObj.port === '443')
  ) {
    urlObj.port = '';
  }

  // 3. Remove trailing slash from path (unless root)
  if (urlObj.pathname !== '/' && urlObj.pathname.endsWith('/')) {
    urlObj.pathname = urlObj.pathname.slice(0, -1);
  }

  // 4. Sort query parameters
  const params = Array.from(urlObj.searchParams.entries())
    .sort((a, b) => a[0].localeCompare(b[0]));
  urlObj.search = '';
  params.forEach(([key, value]) => urlObj.searchParams.append(key, value));

  // 5. Remove fragment (optional)
  urlObj.hash = '';

  // 6. Decode unnecessarily encoded characters
  urlObj.pathname = urlObj.pathname
    .replace(/%2F/g, '/')
    .replace(/%3A/g, ':');

  return urlObj.toString();
}

// Test normalization
const urls = [
  'HTTPS://Example.com/Path?b=2&a=1#section',
  'https://example.com:443/Path/?a=1&b=2',
  'https://example.com/Path?a=1&b=2#other',
  'https://example.com/Path/?a=1&b=2/'
];

const normalized = urls.map(normalizeUrl);
console.log(normalized);
// All become: https://example.com/Path?a=1&b=2

// Check if URLs are equivalent
function urlsAreEquivalent(url1, url2) {
  return normalizeUrl(url1) === normalizeUrl(url2);
}

console.log(urlsAreEquivalent(
  'https://example.com/path?a=1&b=2',
  'https://example.com/path?b=2&a=1'
)); // true

// Compare URLs for equality

function compareUrls(url1, url2, options = {}) {
  const {
    ignoreProtocol = false,
    ignorePort = false,
    ignorePath = false,
    ignoreQuery = false,
    ignoreHash = true,
    ignoreCase = true,
    queryOrder = false  // true = order matters
  } = options;

  const u1 = new URL(url1);
  const u2 = new URL(url2);

  // Compare protocol
  if (!ignoreProtocol) {
    const p1 = ignoreCase ? u1.protocol.toLowerCase() : u1.protocol;
    const p2 = ignoreCase ? u2.protocol.toLowerCase() : u2.protocol;
    if (p1 !== p2) return false;
  }

  // Compare host
  const h1 = ignoreCase ? u1.hostname.toLowerCase() : u1.hostname;
  const h2 = ignoreCase ? u2.hostname.toLowerCase() : u2.hostname;
  if (h1 !== h2) return false;

  // Compare port
  if (!ignorePort && u1.port !== u2.port) {
    return false;
  }

  // Compare path
  if (!ignorePath) {
    const p1 = ignoreCase ? u1.pathname.toLowerCase() : u1.pathname;
    const p2 = ignoreCase ? u2.pathname.toLowerCase() : u2.pathname;
    if (p1 !== p2) return false;
  }

  // Compare query
  if (!ignoreQuery) {
    if (queryOrder) {
      // Order matters
      if (u1.search !== u2.search) return false;
    } else {
      // Order doesn't matter
      const params1 = Array.from(u1.searchParams.entries()).sort();
      const params2 = Array.from(u2.searchParams.entries()).sort();
      if (JSON.stringify(params1) !== JSON.stringify(params2)) {
        return false;
      }
    }
  }

  // Compare hash
  if (!ignoreHash && u1.hash !== u2.hash) {
    return false;
  }

  return true;
}

// Usage
console.log(compareUrls(
  'https://example.com/path?a=1&b=2',
  'https://example.com/path?b=2&a=1'
)); // true (default: ignore query order)

console.log(compareUrls(
  'http://example.com/path',
  'https://example.com/path',
  { ignoreProtocol: false }
)); // false (different protocol)

console.log(compareUrls(
  'https://example.com/path#section1',
  'https://example.com/path#section2',
  { ignoreHash: false }
)); // false (different hash)

// Validate URLs

function isValidUrl(string) {
  try {
    new URL(string);
    return true;
  } catch (error) {
    return false;
  }
}

// More comprehensive validation
function validateUrl(url, options = {}) {
  const {
    allowedProtocols = ['http:', 'https:'],
    requireProtocol = true,
    allowLocalhost = true,
    allowIp = true,
    maxLength = 2048
  } = options;

  // Length check
  if (url.length > maxLength) {
    return { valid: false, error: 'URL too long' };
  }

  // Add protocol if missing and not required
  let testUrl = url;
  if (!requireProtocol && !url.includes('://')) {
    testUrl = `https://${url}`;
  }

  // Parse URL
  let parsed;
  try {
    parsed = new URL(testUrl);
  } catch (error) {
    return { valid: false, error: 'Invalid URL format' };
  }

  // Check protocol
  if (!allowedProtocols.includes(parsed.protocol)) {
    return {
      valid: false,
      error: `Protocol must be one of: ${allowedProtocols.join(', ')}`
    };
  }

  // Check localhost
  if (!allowLocalhost &&
      (parsed.hostname === 'localhost' || parsed.hostname === '127.0.0.1')) {
    return { valid: false, error: 'Localhost URLs not allowed' };
  }

  // Check IP addresses
  if (!allowIp && /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(parsed.hostname)) {
    return { valid: false, error: 'IP addresses not allowed' };
  }

  // Check for suspicious patterns
  if (parsed.username || parsed.password) {
    return { valid: false, error: 'URLs with credentials not allowed' };
  }

  return { valid: true, url: parsed.href };
}

// Usage
console.log(isValidUrl('https://example.com')); // true
console.log(isValidUrl('not a url')); // false

const result = validateUrl('https://example.com/path', {
  allowedProtocols: ['https:'],
  requireProtocol: true
});
console.log(result); // { valid: true, url: 'https://example.com/path' }

// Python equivalent
/*
from urllib.parse import urlparse

def is_valid_url(url):
    try:
        result = urlparse(url)
        return all([result.scheme, result.netloc])
    except:
        return False
*/

// Sanitize URL parameters to prevent XSS

function sanitizeUrlParam(value) {
  if (typeof value !== 'string') {
    return String(value);
  }

  // Remove dangerous characters and patterns
  return value
    .replace(/[<>'"]/g, '')  // Remove HTML/script chars
    .replace(/javascript:/gi, '')  // Remove javascript: protocol
    .replace(/data:/gi, '')  // Remove data: protocol
    .replace(/vbscript:/gi, '')  // Remove vbscript: protocol
    .replace(/on\w+=/gi, '')  // Remove event handlers (onclick=, etc.)
    .trim();
}

function sanitizeUrlParams(url) {
  const urlObj = new URL(url);
  const sanitized = new URL(urlObj.origin + urlObj.pathname);

  // Sanitize each parameter
  for (const [key, value] of urlObj.searchParams) {
    sanitized.searchParams.set(
      sanitizeUrlParam(key),
      sanitizeUrlParam(value)
    );
  }

  return sanitized.toString();
}

// Usage
const dangerousUrl = 'https://example.com/search?q=<script>alert("XSS")</script>&redirect=javascript:alert(1)';
const safe = sanitizeUrlParams(dangerousUrl);
console.log(safe);
// https://example.com/search?q=scriptalertXSSscript&redirect=alert1

// Advanced sanitization with allowlist
function sanitizeWithAllowlist(url, allowedParams) {
  const urlObj = new URL(url);
  const sanitized = new URL(urlObj.origin + urlObj.pathname);

  for (const [key, value] of urlObj.searchParams) {
    // Only allow specified parameters
    if (allowedParams.includes(key)) {
      sanitized.searchParams.set(key, sanitizeUrlParam(value));
    }
  }

  return sanitized.toString();
}

const cleaned = sanitizeWithAllowlist(
  'https://example.com/search?q=test&token=abc&xss=<script>',
  ['q', 'page', 'limit']  // Only allow these params
);
console.log(cleaned);
// https://example.com/search?q=test

// Prevent open redirect vulnerabilities

function isValidRedirect(redirectUrl, currentHost) {
  try {
    const url = new URL(redirectUrl, `https://${currentHost}`);

    // Only allow same-origin redirects
    return url.hostname === currentHost;
  } catch (error) {
    return false;
  }
}

// With allowlist
function isSafeRedirect(redirectUrl, allowedHosts = []) {
  try {
    const url = new URL(redirectUrl);

    // Check if host is in allowlist
    return allowedHosts.some(host => {
      // Support wildcard subdomains
      if (host.startsWith('*.')) {
        const domain = host.slice(2);
        return url.hostname.endsWith(domain);
      }
      return url.hostname === host;
    });
  } catch (error) {
    return false;
  }
}

// Express middleware example
function redirectMiddleware(req, res, next) {
  const redirectUrl = req.query.redirect || req.query.return_url;

  if (!redirectUrl) {
    return next();
  }

  const allowedHosts = [
    'example.com',
    '*.example.com',  // Allow all subdomains
    'trusted-partner.com'
  ];

  if (!isSafeRedirect(redirectUrl, allowedHosts)) {
    return res.status(400).json({
      error: 'Invalid redirect URL'
    });
  }

  res.redirect(redirectUrl);
}

// Usage
console.log(isSafeRedirect(
  'https://app.example.com/dashboard',
  ['example.com', '*.example.com']
)); // true

console.log(isSafeRedirect(
  'https://evil.com?target=example.com',
  ['example.com']
)); // false

// Safe redirect handler
function handleRedirect(redirectUrl, fallbackUrl = '/') {
  const currentHost = window.location.hostname;

  if (isValidRedirect(redirectUrl, currentHost)) {
    window.location.href = redirectUrl;
  } else {
    console.warn('Unsafe redirect blocked:', redirectUrl);
    window.location.href = fallbackUrl;
  }
}

// Popular JavaScript URL manipulation libraries

// 1. URI.js - Comprehensive URL manipulation
// npm install urijs
const URI = require('urijs');

const url = new URI('https://example.com/path?q=test');
url.addQuery('page', 2);
url.segment(1, 'newpath');
console.log(url.toString());

// 2. query-string - Parse and stringify URL query strings
// npm install query-string
const queryString = require('query-string');

const parsed = queryString.parse('?foo=bar&abc=xyz');
console.log(parsed); // { foo: 'bar', abc: 'xyz' }

const stringified = queryString.stringify({ foo: 'bar', abc: 'xyz' });
console.log(stringified); // foo=bar&abc=xyz

// 3. qs - Advanced query string parsing
// npm install qs
const qs = require('qs');

const complex = qs.parse('a[b]=c&a[d]=e');
console.log(complex); // { a: { b: 'c', d: 'e' }}

// 4. url-parse - Universal URL parser
// npm install url-parse
const URLParse = require('url-parse');

const url = new URLParse('https://example.com/path?q=test');
console.log(url.hostname); // example.com

// 5. Native JavaScript (modern browsers/Node.js)
const url = new URL('https://example.com/path?q=test');
const params = new URLSearchParams(url.search);

# Python URL manipulation libraries

# 1. urllib.parse (built-in)
from urllib.parse import urlparse, parse_qs, urlencode, urljoin

url = 'https://example.com/path?q=test&page=2'
parsed = urlparse(url)
print(parsed.scheme)   # https
print(parsed.netloc)   # example.com
print(parsed.path)     # /path

# Parse query string
params = parse_qs(parsed.query)
print(params)  # {'q': ['test'], 'page': ['2']}

# Build query string
query = urlencode({'q': 'test', 'page': 2})
print(query)  # q=test&page=2

# Join URLs
absolute = urljoin('https://example.com/path/', '../other')
print(absolute)  # https://example.com/other

# 2. furl - Fluent URL manipulation
# pip install furl
from furl import furl

url = furl('https://example.com/path')
url.args['q'] = 'test'
url.args['page'] = 2
print(url.url)  # https://example.com/path?q=test&page=2

# 3. yarl - Yet another URL library
# pip install yarl
from yarl import URL

url = URL('https://example.com/path')
url = url.with_query(q='test', page=2)
print(url)  # https://example.com/path?q=test&page=2

# 4. validators - URL validation
# pip install validators
import validators

result = validators.url('https://example.com')
print(result)  # True

# 5. requests - HTTP library with URL utilities
# pip install requests
import requests
from requests.compat import urljoin, quote

url = urljoin('https://api.example.com', '/v1/users')
# https://api.example.com/v1/users