/** * Multi-format parser for Israeli legislation. * * Handles two content formats: * 1. HTML -- from UCI mirror (Privacy Protection Law) * 2. Plain text -- from pdftotext extraction (Computer Law, Basic Law) * * Israeli laws use "Section N" numbering, not "Article N". * Basic Laws use numbered sections without the "Section" prefix in some formats. */ // --------------------------------------------------------------------------- // Interfaces // --------------------------------------------------------------------------- export interface ActIndexEntry { id: string; lawName: string; year: number; title: string; titleEn: string; abbreviation: string; status: 'in_force' | 'amended' | 'repealed' | 'not_yet_in_force'; issuedDate: string; inForceDate: string; url: string; } export interface ParsedProvision { provision_ref: string; chapter?: string; section: string; title: string; content: string; } export interface ParsedDefinition { term: string; definition: string; source_provision?: string; } export interface ParsedAct { id: string; type: 'statute'; title: string; title_en: string; short_name: string; status: 'in_force' | 'amended' | 'repealed' | 'not_yet_in_force'; issued_date: string; in_force_date: string; url: string; description?: string; provisions: ParsedProvision[]; definitions: ParsedDefinition[]; } // --------------------------------------------------------------------------- // HTML utilities // --------------------------------------------------------------------------- function stripHtml(html: string): string { return html .replace(/<[^>]+>/g, ' ') .replace(/ /g, ' ') .replace(/&/g, '&') .replace(/</g, '<') .replace(/>/g, '>') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/\s+/g, ' ') .trim(); } function normalizeText(text: string): string { return text.replace(/\s+/g, ' ').trim(); } // --------------------------------------------------------------------------- // HTML Parser -- for UCI mirror format (Privacy Protection Law) // // Structure: N. Title ...
N+1. Title
// Chapters: CHAPTER ...: ...
// ---------------------------------------------------------------------------
export function parsePrivacyLawHtml(html: string, act: ActIndexEntry): ParsedAct {
const provisions: ParsedProvision[] = [];
const definitions: ParsedDefinition[] = [];
// Extract the law body (inside the main table)
const bodyMatch = html.match(/PROTECTION OF PRIVACY LAW[\s\S]*?(?=<\/TD>\s*<\/TR>\s*<\/TABLE>\s*
)/i);
const body = bodyMatch ? bodyMatch[0] : html;
let currentChapter = '';
// Split by bold section numbers: N. or N.
// Pattern: optionally then section number. title
const sectionPattern = /(?:]*><\/a>)?\s*(\d+[A-Z]?)\.\s+([^<]+)<\/B>/gi;
const chapterPattern = /\s*(CHAPTER\s+[^:]+:\s*[^<]+)<\/B>/gi;
// First, collect chapter positions
const chapters: Array<{ pos: number; name: string }> = [];
let chMatch;
while ((chMatch = chapterPattern.exec(body)) !== null) {
chapters.push({ pos: chMatch.index, name: normalizeText(stripHtml(chMatch[1])) });
}
// Also collect article positions (Article One: Data Bases, Article Two: Direct Mail)
const articlePattern = /\s*(Article\s+[^:]+:\s*[^<]+)<\/B>/gi;
while ((chMatch = articlePattern.exec(body)) !== null) {
chapters.push({ pos: chMatch.index, name: normalizeText(stripHtml(chMatch[1])) });
}
chapters.sort((a, b) => a.pos - b.pos);
// Collect all section matches
const sectionMatches: Array<{ pos: number; num: string; title: string }> = [];
let secMatch;
while ((secMatch = sectionPattern.exec(body)) !== null) {
sectionMatches.push({
pos: secMatch.index,
num: secMatch[1].trim(),
title: normalizeText(stripHtml(secMatch[2])),
});
}
// For each section, determine its chapter and extract content
for (let i = 0; i < sectionMatches.length; i++) {
const sec = sectionMatches[i];
const nextSec = sectionMatches[i + 1];
// Determine chapter for this section
for (const ch of chapters) {
if (ch.pos < sec.pos) {
currentChapter = ch.name;
}
}
// Extract content between this section and next section
const startPos = sec.pos;
const endPos = nextSec ? nextSec.pos : body.length;
const rawContent = body.substring(startPos, endPos);
const content = normalizeText(stripHtml(rawContent));
if (content.length > 10) {
const provRef = `sec${sec.num}`;
provisions.push({
provision_ref: provRef,
chapter: currentChapter || undefined,
section: sec.num,
title: sec.title,
content: content.substring(0, 8000),
});
// Extract definitions from Section 3 and Section 7 (definition sections)
if (sec.num === '3' || sec.num === '7' || sec.num === '17C') {
extractDefinitionsFromContent(content, provRef, definitions);
}
}
}
return {
id: act.id,
type: 'statute',
title: act.title,
title_en: act.titleEn,
short_name: act.abbreviation,
status: act.status,
issued_date: act.issuedDate,
in_force_date: act.inForceDate,
url: act.url,
provisions,
definitions,
};
}
// ---------------------------------------------------------------------------
// Plain-text Parser -- for pdftotext output (Computer Law, Basic Law)
//
// Computer Law format:
// "Section N\n\nTitle text\n\nN. content..."
// or just "N. content..."
//
// Basic Law format:
// "Title label\n\nN. content..."
// ---------------------------------------------------------------------------
export function parseComputerLawText(text: string, act: ActIndexEntry): ParsedAct {
const provisions: ParsedProvision[] = [];
const definitions: ParsedDefinition[] = [];
// The Computer Law PDF from UNODC has a two-part structure:
// 1. Table of Contents (contains "Section N" + "Go" lines)
// 2. Actual law text starting with "Computers Law, 5755"
// We skip the ToC and parse only the actual law text.
const lawTextStart = text.indexOf('Computers Law, 5755');
const lawText = lawTextStart >= 0 ? text.substring(lawTextStart) : text;
let currentChapter = '';
const lines = lawText.split('\n');
const sections: Array<{ num: string; title: string; content: string; chapter: string }> = [];
let currentSection: { num: string; title: string; content: string; chapter: string } | null = null;
// Track marginal note lines (title labels that appear before section numbers)
let marginalNoteLines: string[] = [];
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
// Detect chapter headings
const chapterMatch = line.match(/^Chapter\s+(One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|\w+):\s*(.+)/i);
if (chapterMatch) {
currentChapter = normalizeText(line);
marginalNoteLines = [];
continue;
}
// Pattern 1: "N." alone on a line (section number with content on next line)
// This is the common PDF format where the number is isolated
const sectionAloneMatch = line.match(/^(\d+[A-Za-z]?)\.\s*$/);
if (sectionAloneMatch) {
// Save previous section
if (currentSection) {
sections.push(currentSection);
}
// The marginal note lines before this number are the title
const titleCandidates = marginalNoteLines.filter((l) =>
l.length > 0 && l.length < 100
&& !l.match(/^Chapter\s+/i) && !l.match(/^Go$/i)
&& !l.match(/^Section\s+\d+/) && !l.match(/^Clause\s+/i)
&& !l.match(/^\*/) && !l.match(/^Contents$/)
&& !l.match(/^\d+$/) && !l.match(/^Computers Law/i)
&& !l.match(/^Published in/i)
);
const title = normalizeText(titleCandidates.join(' '));
currentSection = {
num: sectionAloneMatch[1],
title: title,
content: '',
chapter: currentChapter,
};
marginalNoteLines = [];
continue;
}
// Pattern 2: "N. (a) content" or "N. Content text" on the same line
const sectionInlineMatch = line.match(/^(\d+[A-Za-z]?)\.\s+(.+)/);
if (sectionInlineMatch) {
// Check this isn't a page footnote like "* Published in..."
if (sectionInlineMatch[2].match(/^Published in/i)) {
continue;
}
// Save previous section
if (currentSection) {
sections.push(currentSection);
}
// Marginal note = title
const titleCandidates = marginalNoteLines.filter((l) =>
l.length > 0 && l.length < 100
&& !l.match(/^Chapter\s+/i) && !l.match(/^Go$/i)
&& !l.match(/^Section\s+\d+/) && !l.match(/^Clause\s+/i)
&& !l.match(/^\*/) && !l.match(/^Contents$/)
&& !l.match(/^\d+$/) && !l.match(/^Computers Law/i)
);
const title = normalizeText(titleCandidates.join(' '));
currentSection = {
num: sectionInlineMatch[1],
title: title,
content: normalizeText(sectionInlineMatch[0]),
chapter: currentChapter,
};
marginalNoteLines = [];
continue;
}
// Accumulate content for current section
if (currentSection && line.length > 0) {
// Skip page numbers (standalone digits), headers, and footnote markers
if (line.match(/^\d+$/) && line.length <= 3) continue;
if (line.match(/^Computers Law, 1995/i)) continue;
currentSection.content += ' ' + normalizeText(line);
} else if (!currentSection && line.length > 0) {
// Track marginal note lines (before any section starts, or between sections)
if (!line.match(/^Go$/i) && !line.match(/^Section\s+\d+/)
&& !line.match(/^Computers Law/i) && !line.match(/^\d+$/)
&& !line.match(/^\*$/) && !line.match(/^Published in/i)) {
marginalNoteLines.push(line);
} else {
// Reset on non-title lines
if (line.match(/^Go$/i) || line.match(/^Section\s+\d+/)) {
marginalNoteLines = [];
}
}
} else if (line.length === 0 && !currentSection) {
// Empty line resets marginal notes only if we haven't started collecting them recently
// Keep them -- marginal notes can span across blank lines in PDF
}
}
// Save last section
if (currentSection) {
sections.push(currentSection);
}
for (const sec of sections) {
const content = normalizeText(sec.content);
if (content.length > 10) {
provisions.push({
provision_ref: `sec${sec.num}`,
chapter: sec.chapter || undefined,
section: sec.num,
title: sec.title,
content: content.substring(0, 8000),
});
// Extract definitions from Section 1 (uses regular quotes in PDF text)
if (sec.num === '1') {
extractDefinitionsFromPlainText(content, `sec${sec.num}`, definitions);
}
}
}
return {
id: act.id,
type: 'statute',
title: act.title,
title_en: act.titleEn,
short_name: act.abbreviation,
status: act.status,
issued_date: act.issuedDate,
in_force_date: act.inForceDate,
url: act.url,
provisions,
definitions,
};
}
export function parseBasicLawText(text: string, act: ActIndexEntry): ParsedAct {
const provisions: ParsedProvision[] = [];
const definitions: ParsedDefinition[] = [];
// Basic Law format from Knesset PDF:
// "Title label\n\n1.\n\nContent text..."
// or "Title label\n\n1. Content text..."
const lines = text.split('\n');
const sections: Array<{ num: string; title: string; content: string }> = [];
let currentSection: { num: string; title: string; content: string } | null = null;
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
// Detect section start: "N." at start of line
const sectionMatch = line.match(/^(\d+[a-z]?)\.\s*(.*)/);
if (sectionMatch) {
// Look back for title (marginal label)
let title = '';
for (let j = i - 1; j >= Math.max(0, i - 4); j--) {
const prevLine = lines[j].trim();
if (prevLine.length > 0 && !prevLine.match(/^\d+[a-z]?\.\s/)
&& !prevLine.match(/^\(Amendment/) && prevLine.length < 100) {
title = prevLine + (title ? ' ' + title : '');
} else if (prevLine.length === 0 && title.length > 0) {
break;
}
}
// Save previous section
if (currentSection) {
sections.push(currentSection);
}
currentSection = {
num: sectionMatch[1],
title: normalizeText(title),
content: sectionMatch[2] ? normalizeText(sectionMatch[0]) : '',
};
continue;
}
// Accumulate content
if (currentSection && line.length > 0) {
// Skip header / footer lines
if (line.match(/^BASIC-LAW:/i)) continue;
if (line.match(/^This unofficial/i)) continue;
if (line.match(/^For the full/i)) continue;
if (line.match(/^Special thanks/i)) continue;
currentSection.content += ' ' + normalizeText(line);
}
}
if (currentSection) {
sections.push(currentSection);
}
for (const sec of sections) {
const content = normalizeText(sec.content);
if (content.length > 10) {
provisions.push({
provision_ref: `sec${sec.num}`,
chapter: undefined,
section: sec.num,
title: sec.title,
content: content.substring(0, 8000),
});
}
}
return {
id: act.id,
type: 'statute',
title: act.title,
title_en: act.titleEn,
short_name: act.abbreviation,
status: act.status,
issued_date: act.issuedDate,
in_force_date: act.inForceDate,
url: act.url,
provisions,
definitions,
};
}
// ---------------------------------------------------------------------------
// Definition extractors
// ---------------------------------------------------------------------------
function extractDefinitionsFromContent(
content: string,
sourceProvision: string,
definitions: ParsedDefinition[],
): void {
// Pattern: "term" - definition text; or "term" has the meaning...
const defPattern = /["\u201c]([^"\u201d]+)["\u201d]\s*[-\u2013\u2014]\s*([^;]+(?:;|$))/g;
let match;
while ((match = defPattern.exec(content)) !== null) {
const term = normalizeText(match[1]);
const definition = normalizeText(match[2]).replace(/;$/, '').trim();
if (term.length > 1 && term.length < 80 && definition.length > 5) {
// Avoid duplicates
if (!definitions.some((d) => d.term === term)) {
definitions.push({ term, definition, source_provision: sourceProvision });
}
}
}
}
function extractDefinitionsFromPlainText(
content: string,
sourceProvision: string,
definitions: ParsedDefinition[],
): void {
// Computer Law definitions format (PDF uses regular quotes):
// "computer material" - software or information;
// Also handle curly quotes from other sources
const patterns = [
/["\u201c]([^"\u201d]+)["\u201d]\s*[-\u2013\u2014]+\s*([^;]+;)/g,
/"([^"]+)"\s*[-\u2013\u2014]+\s*([^;]+;)/g,
];
const seen = new Set