# Compares every translation of the three books with the English, block by block, so a translation
# that has fallen behind the English, or lost or broken something on the way, is found before it
# ships.  The books are docs/Manual.adoc, docs/Reference.adoc, and docs/Learn.adoc (with the lessons
# it includes);  each language has the same files under docs/<language>/.  For each it checks:
#
#   * the same sections, with the same ids, levels, and kinds, in the same order (a translation
#     writes the English id of every section whose id Asciidoctor would make from its title);
#   * within each section, the same blocks in the same order:  paragraphs, lists with as many items,
#     tables with as many rows and columns, the same pictures, notes of the same kind, and listings
#     in the same language;
#   * Lua listings identical once their comments are taken out (only comments are translated), and
#     every other listing with as many lines;
#   * within each section, the same references (<<id>> and xref:Book.adoc#id), as many times each;
#   * that Asciidoctor finds nothing wrong in the translation it does not find in the English;
#   * that the English is ASCII and the translation names its language (:lang:) and has its captions.
#
# Inline code (`spans`) and web addresses that differ within a section are warnings;  everything else
# is an error.  Exits 1 on any error.
#
#   ruby util/checkTranslations.rb [language...]                   every language, or those named
#   ruby util/checkTranslations.rb --pair english.adoc translated.adoc
#                                                                 two pieces of a book, as a
#                                                                 translator checks a part
require 'asciidoctor'

LANGUAGES = %w[fr it de es]
BOOKS     = %w[Manual Reference Learn]
REPO      = File.dirname(__dir__)
DOCS      = File.join(REPO, 'docs')
REPORTED  = 12   # Differences reported per section before the rest are only counted


# Asciidoctor's warnings for a document, without their line numbers and with only the last part of
# any path, so the English's and a translation's can be compared.
def warningTexts(logger)
	logger.messages.select { |entry| entry[:severity] != :INFO && entry[:severity] != :DEBUG }.map do |entry|
		text = entry[:message]
		text = text[:text] if text.is_a?(Hash)
		text.to_s.gsub(%r{(?:[^\s:]*/)+([^\s/]+)}, '\1')
	end
end


# Loads a book or a piece of one with every block's source line, and its warnings.
def loadBook(path)
	logger                             = Asciidoctor::MemoryLogger.new
	Asciidoctor::LoggerManager.logger = logger
	doc                                = Asciidoctor.load_file(path, safe: :unsafe, sourcemap: true, attributes: { 'revnumber' => 'x', 'doctype' => 'book' })
	[doc, warningTexts(logger)]
end


# A Lua listing with its comments taken out and blank and trailing space dropped:  what a
# translation must leave exactly as it is.
def luaWithoutComments(code)
	out   = +''
	index = 0
	while index < code.length
		char = code[index]
		if char == '-' && code[index + 1] == '-'
			long = code[(index + 2)..].match(/\A\[(=*)\[/)
			if long
				close = "]#{long[1]}]"
				found = code.index(close, index + 2 + long[0].length)
				index = found ? found + close.length : code.length
			else
				found = code.index("\n", index)
				index = found || code.length
			end
		elsif char == '"' || char == "'"
			found = index + 1
			while found < code.length && code[found] != char && code[found] != "\n"
				found += (code[found] == '\\') ? 2 : 1
			end
			out << code[index..found].to_s
			index = found + 1
		elsif char == '[' && (long = code[index..].match(/\A\[(=*)\[/))
			close = "]#{long[1]}]"
			found = code.index(close, index + long[0].length)
			found = found ? found + close.length : code.length
			out << code[index...found]
			index = found
		else
			out << char
			index += 1
		end
	end
	out.lines.map(&:rstrip).reject(&:empty?).join("\n")
end


# What a translation must keep of a block.
def blockKey(node)
	key = node.context.to_s
	case node.context
	when :section
		key = "section #{node.level} #{node.id} #{node.sectname}"
	when :listing, :literal
		language = node.attr('language')
		if language == 'lua'
			key = "#{node.context} lua\n#{luaWithoutComments(node.source)}"
		else
			key = "#{node.context} #{language} #{node.source.lines.count} lines"
		end
	when :image
		key = "image #{node.attr('target')}"
	when :table
		key = "table #{node.rows.head.size}+#{node.rows.body.size}+#{node.rows.foot.size} rows #{node.columns.size} columns"
	when :ulist, :olist, :colist, :dlist
		key = "#{node.context} of #{node.items.size}"
	when :admonition
		key = "admonition #{node.attr('name')}"
	when :floating_title
		key = "heading #{node.level}"
	end
	if node.context != :section && node.id && !node.id.start_with?('_')
		key += " ##{node.id}"
	end
	key
end


# Every block of a document in order, grouped by the section it is in.  Returns the sections' ids
# in order and, for each, its blocks' keys and source places.
def sections(doc)
	order  = ['(top)']
	groups = { '(top)' => [] }
	places = { '(top)' => [] }
	starts = { '(top)' => nil }
	current = '(top)'
	doc.find_by(traverse_documents: true).each do |node|
		next if node.context == :document || node.context == :list_item
		if node.context == :section
			current = node.id
			order << current
			groups[current] = []
			places[current] = []
			starts[current] = node.source_location
		end
		groups[current] << blockKey(node)
		places[current] << node.source_location
	end
	[order, groups, places, starts]
end


# The text of each section as written, from its title to the next section's in the same file,
# without its listings:  where its references and inline code are counted.
def sectionTexts(starts)
	texts = {}
	byFile = starts.select { |_, place| place }.group_by { |_, place| place.file }
	byFile.each do |file, list|
		next unless file && File.exist?(file)
		lines = File.readlines(file)
		list  = list.sort_by { |_, place| place.lineno }
		list.each_with_index do |(id, place), index|
			last    = (index + 1 < list.size) ? list[index + 1][1].lineno - 1 : lines.size
			inBlock = nil
			text    = +''
			lines[(place.lineno - 1)...last].each do |line|
				fence = line.rstrip
				if fence.match?(/\A(-{4,}|\.{4,}|\+{4,})\z/)
					inBlock = (inBlock == fence) ? nil : (inBlock || fence)
					next
				end
				text << line unless inBlock
			end
			texts[id] = text
		end
	end
	texts
end


def counts(list)
	list.each_with_object(Hash.new(0)) { |item, total| total[item] += 1 }
end


# The references, inline code, and web addresses of a section's text.
def inlineParts(text)
	references = text.scan(/<<([^,>]+)/).flatten.map(&:strip) + text.scan(/xref:([^\[\s]+)\[/).flatten
	# Asciidoctor's own rule for constrained monospace, so a span it will not render (l'`id`, an
	# apostrophe before the backtick) is not counted as one.
	code       = text.scan(/(?:^|(?<=[^\p{Word};:"'`}\\+]))`(\S|\S[^`]{0,200}?\S)`(?![\p{Word}"'`])/m).flatten.map { |span| span.gsub(/\s+/, ' ') }
	addresses  = text.scan(%r{https?://[^\s\[\]<>"]+}).map { |address| address.sub(/[.,;:)]+\z/, '') }
	[counts(references), counts(code), counts(addresses)]
end


def where(place)
	place ? "#{place.file.sub("#{REPO}/", '')}:#{place.lineno}" : '?'
end


def describe(key)
	first = key.lines.first.to_s.strip
	key.include?("\n") ? "#{first} (#{key.lines.count - 1} lines of code)" : first
end


def differences(english, translated)
	(english.keys | translated.keys).filter_map do |item|
		gap = translated[item] - english[item]
		gap.zero? ? nil : "#{item} #{gap.positive? ? '+' : ''}#{gap}"
	end
end


# Compares one book or piece with its translation.  Returns the errors and warnings.
def compare(englishPath, translatedPath, language)
	errors   = []
	warnings = []
	english, englishWarnings       = loadBook(englishPath)
	translated, translatedWarnings = loadBook(translatedPath)

	File.read(englishPath).each_line.with_index do |line, index|
		errors << "#{where(Asciidoctor::Reader::Cursor.new(englishPath, nil, nil, index + 1))}:  the English is not ASCII" unless line.ascii_only?
	end
	if language
		errors << "#{translatedPath}:  :lang: is #{translated.attr('lang').inspect}, not #{language}" if translated.attr('lang') != language
		errors << "#{translatedPath}:  the captions are English;  include::locale.adoc[] is missing" if translated.attr('toc-title') == english.attr('toc-title')
	end
	gap = counts(translatedWarnings)
	counts(englishWarnings).each { |text, count| gap[text] -= count }
	gap.each { |text, count| errors << "#{translatedPath}:  Asciidoctor warns, where the English does not:  #{text}" if count.positive? }

	englishOrder, englishGroups, englishPlaces, englishStarts = sections(english)
	translatedOrder, translatedGroups, translatedPlaces, translatedStarts = sections(translated)
	missing = englishOrder - translatedOrder
	extra   = translatedOrder - englishOrder
	missing.each { |id| errors << "#{where(englishStarts[id])}:  section #{id} is not in the translation" }
	extra.each { |id| errors << "#{where(translatedStarts[id])}:  section #{id} is not in the English" }
	if missing.empty? && extra.empty? && englishOrder != translatedOrder
		at = englishOrder.zip(translatedOrder).index { |a, b| a != b }
		errors << "#{where(translatedStarts[translatedOrder[at]])}:  the sections are in a different order from here (the English has #{englishOrder[at]})"
	end

	englishTexts    = sectionTexts(englishStarts)
	translatedTexts = sectionTexts(translatedStarts)
	(englishOrder & translatedOrder).each do |id|
		ours   = englishGroups[id]
		theirs = translatedGroups[id]
		if ours != theirs
			at = (0...[ours.size, theirs.size].max).find { |index| ours[index] != theirs[index] }
			if ours[at] && theirs[at] && ours[at].include?("\n") && theirs[at].include?("\n")
				line   = ours[at].lines.zip(theirs[at].lines).index { |a, b| a != b } || 0
				detail = "the code differs at its line #{line}:  English #{ours[at].lines[line].to_s.strip.inspect}, translation #{theirs[at].lines[line].to_s.strip.inspect}"
			else
				detail = "English has #{ours[at] ? describe(ours[at]) : 'nothing more'} (#{where(englishPlaces[id][at])}), translation has #{theirs[at] ? describe(theirs[at]) : 'nothing more'}"
			end
			errors << "#{where(translatedPlaces[id][at] || translatedStarts[id])}:  in section #{id}, block #{at + 1}:  #{detail}"
		end
		next unless englishTexts[id] && translatedTexts[id]
		ourReferences, ourCode, ourAddresses       = inlineParts(englishTexts[id])
		theirReferences, theirCode, theirAddresses = inlineParts(translatedTexts[id])
		gaps = differences(ourReferences, theirReferences)
		errors << "#{where(translatedStarts[id])}:  in section #{id}, references differ:  #{gaps.join(',  ')}" unless gaps.empty?
		gaps = differences(ourCode, theirCode)
		warnings << "#{where(translatedStarts[id])}:  in section #{id}, inline code differs:  #{gaps.first(REPORTED).join(',  ')}#{gaps.size > REPORTED ? ",  and #{gaps.size - REPORTED} more" : ''}" unless gaps.empty?
		gaps = differences(ourAddresses, theirAddresses)
		warnings << "#{where(translatedStarts[id])}:  in section #{id}, web addresses differ:  #{gaps.join(',  ')}" unless gaps.empty?
	end
	[errors, warnings]
end


def report(errors, warnings)
	errors.each { |text| puts "error:  #{text}" }
	warnings.each { |text| puts "warning:  #{text}" }
end


def main
	errors   = []
	warnings = []
	if ARGV.first == '--pair'
		abort 'usage:  checkTranslations.rb --pair english.adoc translated.adoc' unless ARGV.size == 3
		errors, warnings = compare(File.expand_path(ARGV[1]), File.expand_path(ARGV[2]), nil)
	else
		languages = ARGV.empty? ? LANGUAGES : ARGV
		languages.each do |language|
			abort "unknown language #{language}" unless LANGUAGES.include?(language)
			lessons = Dir.glob(File.join(DOCS, 'lessons', '*.adoc')).map { |path| File.join('lessons', File.basename(path)) }
			lessons.each do |lesson|
				errors << "docs/#{language}/#{lesson}:  missing" unless File.exist?(File.join(DOCS, language, lesson))
			end
			BOOKS.each do |book|
				translated = File.join(DOCS, language, "#{book}.adoc")
				if File.exist?(translated)
					found = compare(File.join(DOCS, "#{book}.adoc"), translated, language)
					errors.concat(found[0])
					warnings.concat(found[1])
				else
					errors << "docs/#{language}/#{book}.adoc:  missing"
				end
			end
		end
	end
	report(errors, warnings)
	puts "#{errors.size} errors, #{warnings.size} warnings"
	exit(errors.empty? ? 0 : 1)
end


main