Test system resources for CLUSTER case in model_maker (d25e91ae) · Commits · Giacomo Mulas / NP_TMcode

src/scripts/model_maker.py

+99 −35

Original line number	Diff line number	Diff line
		@@ -441,41 +441,7 @@ def load_model(model_file):
		if (max_rad == 0.0):
		max_rad = 20.0 * max(sconf['ros'])
		write_obj(sconf, gconf, max_rad)
		try:
		max_gpu_ram = int(model['system_settings']['max_gpu_ram'])
		matrix_dim = 2 * gconf['nsph'] * gconf['li'] * (gconf['li'] + 2)
		matrix_size_bytes = 16 * matrix_dim * matrix_dim
		matrix_size_Gb = float(matrix_size_bytes) / 1024.0 / 1024.0 / 1024.0
		print("INFO: estimated matrix size is {0:.3g} Gb.".format(matrix_size_Gb))
		if (max_gpu_ram > 0):
		max_gpu_ram_bytes = max_gpu_ram * 1024 * 1024 * 1024
		if (matrix_size_bytes < max_gpu_ram_bytes):
		max_gpu_processes = int(max_gpu_ram_bytes / matrix_size_bytes)
		print("INFO: system supports up to %d simultaneous processes on GPU."%max_gpu_processes)
		print("INFO: only %d GPU processes allowed, if using refinement."%(max_gpu_processes / 3))
		else:
		print("WARNING: estimated matrix size is larger than available GPU memory!")
		else:
		print("INFO: no GPU RAM declared.")
		max_host_ram = int(model['system_settings']['max_host_ram'])
		if (max_host_ram > 0):
		max_host_ram_bytes = max_host_ram * 1024 * 1024 * 1024
		matrix_dim = 2 * gconf['le'] * (gconf['le'] + 2)
		matrix_size_bytes = 16 * matrix_dim * matrix_dim
		matrix_size_Gb = float(matrix_size_bytes) / 1024.0 / 1024.0 / 1024.0
		print("INFO: estimated external matrix size is {0:.3g} Gb.".format(matrix_size_Gb))
		if (matrix_size_bytes < max_host_ram_bytes):
		max_host_processes = int(max_host_ram_bytes / matrix_size_bytes / 2)
		print("INFO: system supports up to %d simultaneous processes."%max_host_processes)
		else:
		print("WARNING: estimated matrix size is larger than available host memory!")
		else:
		print("WARNING: no host RAM declared!")
		except KeyError as ex:
		print(ex)
		print("WARNING: missing system description! Cannot estimate recommended execution.")
		cpu_count = multiprocessing.cpu_count()
		print("INFO: the number of detected CPUs is %d."%cpu_count)
		test_system_resources(model, gconf, sconf)
		else: # model is None
		print("ERROR: could not parse " + model_file + "!")
		return (sconf, gconf)
		@@ -894,6 +860,104 @@ def random_compact(scatterer, geometry, seed, max_rad):
		sph_index += 1
		return current_n

		def test_system_resources(model, gconf, sconf):
		le = gconf['le'] if (gconf['application'] != "SPH") else 0
		li = gconf['li']
		lm = le if le > li else li
		nsph = gconf['nsph']
		nlim = li * (li + 2)
		nlem = le * (le + 2)
		nlemt = 2 * nlem
		ncou = nsph * nsph - 1
		litpo = li + li + 1
		litpos = litpo * litpo
		lmpo = lm + 1
		lmtpo = li + le + 1
		lmtpos = lmtpo * lmtpo
		nv3j = (lm * (lm + 1) * (2 * lm + 7)) / 6;
		ndi = nsph * nlim
		ndit = 2 * ndi
		nllt = 2 * nsph * li * (li + 2) if nlemt == 0 else nlemt
		npnt = gconf['npnt']
		npntts = gconf['npntts']
		max_n = npnt if npnt > npntts else npntts
		nhspo = 2 * max_n - 1
		configurations = sconf['configurations']
		num_layers = 0
		max_layers = 1
		for nli in range(configurations):
		nl = sconf['nshl'][nli]
		if (nli == 0 and gconf['application'] == "INCLU"): nl += 1
		num_layers += nl
		if (nl > max_layers): max_layers = nl
		try:
		max_gpu_ram = int(model['system_settings']['max_gpu_ram'])
		matrix_dim = 2 * gconf['nsph'] * gconf['li'] * (gconf['li'] + 2)
		matrix_size_bytes = 16 * matrix_dim * matrix_dim
		matrix_size_Gb = float(matrix_size_bytes) / 1024.0 / 1024.0 / 1024.0
		print("INFO: estimated matrix size is {0:.3g} Gb.".format(matrix_size_Gb))
		if (max_gpu_ram > 0):
		max_gpu_ram_bytes = max_gpu_ram * 1024 * 1024 * 1024
		if (matrix_size_bytes < max_gpu_ram_bytes):
		max_gpu_processes = int(max_gpu_ram_bytes / matrix_size_bytes)
		print("INFO: system supports up to %d simultaneous processes on GPU."%max_gpu_processes)
		print("INFO: only %d GPU processes allowed, if using refinement."%(max_gpu_processes / 3))
		else:
		print("WARNING: estimated matrix size is larger than available GPU memory!")
		else:
		print("INFO: no GPU RAM declared.")
		max_host_ram = int(model['system_settings']['max_host_ram'])
		max_host_ram_bytes = max_host_ram * 1024 * 1024 * 1024
		if (max_host_ram > 0):
		# ClusterIterationData section
		required_ram_bytes = 8 * 37
		required_ram_bytes += 8 * 10
		required_ram_bytes += 16
		required_ram_bytes += 4 * 7
		required_ram_bytes += 1
		required_ram_bytes += 8 * (44 + 6 * lm + ndit)
		required_ram_bytes += 8 * (99 + 5 * nsph + 12 * lm)
		required_ram_bytes += 16 * (24 + 4 * nsph + ndit * ndit)
		# ParticleDescriptorCluster section
		required_ram_bytes += 2 * 2
		required_ram_bytes += 4 * 16
		required_ram_bytes += 8 * 20
		required_ram_bytes += 8
		required_ram_bytes += 16 * (2 * li * nsph + 16 + 2 * nhspo + nsph + max_layers + 1)
		required_ram_bytes += 16 * (2 * li + configurations)
		required_ram_bytes += 8 * (num_layers + configurations + 4 * nsph)
		required_ram_bytes += 4 * (nsph + configurations)
		required_ram_bytes += 8 * 12
		required_ram_bytes += 16 * 22 * nsph
		required_ram_bytes += 8 * 7 * nsph
		required_ram_bytes += 8 * 4 * nsph
		required_ram_bytes += 8 * 9
		required_ram_bytes += 16
		required_ram_bytes += 8 * 3
		required_ram_bytes += 16 * (20 + 2 * ndi * nlem + ndit * nlemt)
		required_ram_bytes += 8 * (2 + 2 * ndit)
		required_ram_bytes += 4 * 29
		required_ram_bytes += 8 * 25
		required_ram_bytes += 16
		required_ram_bytes += 16 * (
		4 * nllt + nlemt * nlemt + 36 + ncou * litpo + nsph * lmtpo
		+ ncou * litpos + nsph * lmtpos
		)
		required_ram_gb = required_ram_bytes / 1024.0 / 1024.0 / 1024.0
		print("INFO: model requires %.5gGb of host RAM."%required_ram_gb)
		if (required_ram_bytes < max_host_ram_bytes):
		max_host_processes = int(max_host_ram_bytes / required_ram_bytes)
		print("INFO: system supports up to %d simultaneous processes."%max_host_processes)
		else:
		print("WARNING: estimated matrix size is larger than available host memory!")
		else:
		print("WARNING: no host RAM declared!")
		except KeyError as ex:
		print(ex)
		print("WARNING: missing system description! Cannot estimate recommended execution.")
		cpu_count = multiprocessing.cpu_count()
		print("INFO: the number of detected CPUs is %d."%cpu_count)

		## \brief Write the geometry configuration dictionary to legacy format.
		#
		# \param conf: `dict` Geometry configuration dictionary.