Enable line-wise pycompare option for big files. (ea91f54f) · Commits · Giacomo Mulas / NP_TMcode

src/scripts/pycompare.py

+108 −51

Original line number	Original line	Diff line number	Diff line
	@@ -19,6 +19,7 @@
	# The script execution requires python3.		# The script execution requires python3.

	import re		import re
			import os

	from math import log10		from math import log10
	from sys import argv		from sys import argv
	@@ -99,15 +100,44 @@ def compare_files(config):
	fortran_file = open(config['fortran_file_name'], 'r')		fortran_file = open(config['fortran_file_name'], 'r')
	c_file = open(config['c_file_name'], 'r')		c_file = open(config['c_file_name'], 'r')
	l_file = None		l_file = None
			f_lines = []
			c_lines = []
			line_count = 0
			if (not config['linewise']):
	f_lines = fortran_file.readlines()		f_lines = fortran_file.readlines()
	c_lines = c_file.readlines()		c_lines = c_file.readlines()
			line_count = len(f_lines)
	fortran_file.close()		fortran_file.close()
	c_file.close()		c_file.close()
	if (len(f_lines) == len(c_lines)):		else: # line-wise comparison mode
	line_count = len(f_lines)		f_lines = [fortran_file.readline()]
	num_len = 1		c_lines = [c_file.readline()]
	if (line_count > 0):		print("INFO: using line-wise mode")
	num_len = max(4, int(log10(line_count)) + 1)		print("INFO: counting result lines...")
			while (f_lines[0] != ''):
			if (c_lines[0] != ''):
			line_count += 1
			else:
			print("ERROR: C++ file is shorter than FORTRAN file.")
			fortran_file.close()
			c_file.close()
			mismatch_count['errors'] = line_count
			return mismatch_count
			f_lines[0] = fortran_file.readline()
			c_lines[0] = c_file.readline()
			if (c_lines[0] != ''):
			print("ERROR: C++ file is longer than FORTRAN file.")
			fortran_file.close()
			c_file.close()
			mismatch_count['errors'] = line_count
			return mismatch_count
			fortran_file.close()
			c_file.close()
			print("INFO: the output files have %d lines"%line_count)
			fortran_file = open(config['fortran_file_name'], 'r')
			c_file = open(config['c_file_name'], 'r')
			num_read_lines = 0
			# LOG FILE INITIALIZATION #
	if (config['log_html']):		if (config['log_html']):
	l_file = open(config['html_output'], 'w')		l_file = open(config['html_output'], 'w')
	l_file.write("<!DOCTYPE html>\n")		l_file.write("<!DOCTYPE html>\n")
	@@ -124,7 +154,22 @@ def compare_files(config):
	+ "GREEN</span>, warnings are marked <span style=\"font-weight: bold; color: rgb(0,0,255)\">"		+ "GREEN</span>, warnings are marked <span style=\"font-weight: bold; color: rgb(0,0,255)\">"
	+ "BLUE</span> and errors are marked <span style=\"font-weight: bold; color: rgb(255,0,0)\">"		+ "BLUE</span> and errors are marked <span style=\"font-weight: bold; color: rgb(255,0,0)\">"
	+ "RED</span>.</div>\n")		+ "RED</span>.</div>\n")
	for li in range(line_count):		# END LOG FILE INITIALIZATION #
			line_loop = True
			num_len = 1
			if (line_count > 0):
			num_len = max(4, int(log10(line_count)) + 1)
			print("INFO: checking file contents...")
			while (line_loop):
			if (not config['linewise']):
			line_loop = False
			else:
			f_lines = [fortran_file.readline()]
			c_lines = [c_file.readline()]
			num_read_lines += 1
			# Start here the comparison loop
			if (len(f_lines) == len(c_lines)):
			for li in range(len(f_lines)):
	line_result = compare_lines(f_lines[li], c_lines[li], config, li + 1, num_len, l_file)		line_result = compare_lines(f_lines[li], c_lines[li], config, li + 1, num_len, l_file)
	mismatch_count['errors'] += line_result[0]		mismatch_count['errors'] += line_result[0]
	mismatch_count['warnings'] += line_result[1]		mismatch_count['warnings'] += line_result[1]
	@@ -132,10 +177,6 @@ def compare_files(config):
	if (mismatch_count['errors'] > 0 and not config['check_all']):		if (mismatch_count['errors'] > 0 and not config['check_all']):
	print("INFO: mismatch found at line %d"%(li + 1))		print("INFO: mismatch found at line %d"%(li + 1))
	break		break
	if l_file is not None:
	l_file.write(" </body>\n")
	l_file.write("</html>\n")
	l_file.close()
	else:		else:
	mismatch_count['errors'] = len(c_lines)		mismatch_count['errors'] = len(c_lines)
	print("ERROR: {0:s} and {1:s} have different numbers of lines!".format(		print("ERROR: {0:s} and {1:s} have different numbers of lines!".format(
	@@ -144,6 +185,13 @@ def compare_files(config):
	if (config['log_html']):		if (config['log_html']):
	print("Different file sizes. No log produced.")		print("Different file sizes. No log produced.")
	config['log_html'] = False		config['log_html'] = False
			if (num_read_lines >= line_count):
			line_loop = False
			#End line loop
			if l_file is not None:
			l_file.write(" </body>\n")
			l_file.write("</html>\n")
			l_file.close()
	return mismatch_count		return mismatch_count

	## \brief Perform the comparison of two file lines.		## \brief Perform the comparison of two file lines.
	@@ -381,6 +429,7 @@ def parse_arguments():
	'fortran_file_name': '',		'fortran_file_name': '',
	'c_file_name': '',		'c_file_name': '',
	'full_log': False,		'full_log': False,
			'linewise': False,
	'log_html': False,		'log_html': False,
	'html_output': 'pycompare.html',		'html_output': 'pycompare.html',
	'warning_threshold': 0.005,		'warning_threshold': 0.005,
	@@ -403,6 +452,8 @@ def parse_arguments():
	config['warning_threshold'] = float(split_arg[1])		config['warning_threshold'] = float(split_arg[1])
	elif (arg.startswith("--help")):		elif (arg.startswith("--help")):
	config['help_mode'] = True		config['help_mode'] = True
			elif (arg.startswith("--linewise")):
			config['linewise'] = True
	elif (arg.startswith("--quick")):		elif (arg.startswith("--quick")):
	config['check_all'] = False		config['check_all'] = False
	else:		else:
	@@ -424,6 +475,7 @@ def print_help():
	print("--full Print all lines to log file (default prints only mismatches).")		print("--full Print all lines to log file (default prints only mismatches).")
	print("--help Print this help and exit.")		print("--help Print this help and exit.")
	print("--html[=OPT_OUTPUT_NAME] Enable logging to HTML file (default logs to \"pycompare.html\").")		print("--html[=OPT_OUTPUT_NAME] Enable logging to HTML file (default logs to \"pycompare.html\").")
			print("--linewise Load only one line at a time. Useful to compare big files (false by default).")
	print("--quick Stop on first mismatch (default is to perform a full check).")		print("--quick Stop on first mismatch (default is to perform a full check).")
	print("--warn Set a fractional threshold for numeric warning (default = 0.005).")		print("--warn Set a fractional threshold for numeric warning (default = 0.005).")
	print(" ")		print(" ")
	@@ -440,20 +492,25 @@ def print_help():
	# \param noisy: `int` The number of noisy values detected by the comparison.		# \param noisy: `int` The number of noisy values detected by the comparison.
	def reformat_log(config, errors, warnings, noisy):		def reformat_log(config, errors, warnings, noisy):
	log_file = open(config['html_output'], 'r')		log_file = open(config['html_output'], 'r')
	log_lines = log_file.readlines()		new_file = open("PYCOMPARE_TEMPORARY_LOG.html", 'w')
	log_file.close()		for hi in range(7):
	log_file = open(config['html_output'], 'w')		log_line = log_file.readline()
	for i in range(7): log_file.write(log_lines[i] + "\n")		new_file.write(log_line)
	str_errors = "error" if errors == 1 else "errors"		str_errors = "error" if errors == 1 else "errors"
	str_warnings = "warning" if warnings == 1 else "warnings"		str_warnings = "warning" if warnings == 1 else "warnings"
	str_noisy = "noisy value" if noisy == 1 else "noisy values"		str_noisy = "noisy value" if noisy == 1 else "noisy values"
	summary = " <div>Comparison yielded %d %s"%(errors, str_errors)		summary = " <div>Comparison yielded %d %s"%(errors, str_errors)
	summary = summary + ", %d %s"%(warnings, str_warnings)		summary = summary + ", %d %s"%(warnings, str_warnings)
	summary = summary + " and %d %s.</div>\n"%(noisy, str_noisy)		summary = summary + " and %d %s.</div>\n"%(noisy, str_noisy)
	log_file.write(summary)		new_file.write(summary)
	for i in range(7, len(log_lines)): log_file.write(log_lines[i] + "\n")		log_line = log_file.readline()
			while (log_line != ''):
			new_file.write(log_line)
			log_line = log_file.readline()
	log_file.close()		log_file.close()
			new_file.close()
			os.remove(config['html_output'])
			os.rename("PYCOMPARE_TEMPORARY_LOG.html", config['html_output'])

	# ### PROGRAM EXECUTION ###		# ### PROGRAM EXECUTION ###
	## \cond		## \cond