Commit 5491a0f3 authored by Giuseppe Carboni's avatar Giuseppe Carboni
Browse files

Major Makefile rework 

For more information check the Makefile for useful comments about the major refactoring.
With a 12 cores virtual machine, compiling DISCOS with this Makefile using `time make -j` result in less than 14 minutes of real time. Quite a big leap from the 40+ minutes that a sequential build takes.
parent 2478ea9e

SystemMake/Makefile

+130 −155
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ 
# Bartolini 02/02/15  added .PHONY target tall

# Bartolini 03/02/15  split away common, inspired by http://lackof.org/taggart/hacking/make-example/

# Bartolini 25/08/15  Added comments and changed default target for all

# Carboni   14/04/25  Enabled correct handling of parallel jobs (-j flag)

# Carboni   20/04/25  Parallelized code compilation with make -j



#

# this new Makefile section separates code modules between what is common
@@ -21,18 +21,36 @@ 
# and there is no need to replicate common code modules for all telescopes.

#



# Documentation under Common directory

COMMON_DOC:=

# Common errors, can all be compiled in parallel

COMMON_ERRORS:=ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors XBackendErrors ActiveSurfaceErrors

COMMON_INTERFACES:=CommonInterface ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface XBackendInterface

COMMON_LIBRARIES:=SlaLibrary IRALibrary DiscosVersion TextWindowLibrary ParserLibrary XarcosLibrary ModbusChannel ComponentProxy DiscosLocals XEphemAstroLib DiscosBackendProtocol PyTestingLibrary

# The CommonInterface is the only interface that has no other dependency, must be compiled first

BASE_INTERFACES:=CommonInterface

# The following core interfaces has no dependency between them, they only depend on CommonInterface, therefore they can be compiled in parallel

CORE_INTERFACES:=ManagmentInterface AntennaInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface

# These common interfaces depends on some interface listed in the line above, must be compiled separately

EXTENDED_INTERFACES:=ReceiversInterface XBackendInterface

# These libraries must be compiled sequentially, since the DiscosVersion library depends on the IRALibrary which depends on the SlaLibrary

COMMON_SEQ_LIBS:=SlaLibrary IRALibrary DiscosVersion

# All these other libraries only depend on the libraries listed above, therefore they can be compiled in parallel

COMMON_LIBRARIES:=TextWindowLibrary ParserLibrary XarcosLibrary ModbusChannel ComponentProxy DiscosLocals XEphemAstroLib DiscosBackendProtocol PyTestingLibrary

# All the common servers are independend from one another, they only depend on interfaces, therefore we can compile them in parallel.

# There may be some hidden dependency between servers, but it may never come up since each core compiles a single cpp file.

# Issues may only arise if we use a very high number of cores, which is unlikely.

# If we'll ever face this problem we will split the SERVERS into more groups, with the latter one having servers which depends on some other servers in the first group.

COMMON_SERVERS:=AntennaBoss Observatory OTF PointingModel Refraction SkySource SolarSystem Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool TotalPower CustomLogger \

				PyDewarPositioner Sardara Skarab PyLocalOscillator MFKBandBaseReceiver PyCalmux ActiveSurfaceLan ActiveSurfaceUSD XBackend

# Same as servers

COMMON_CLIENTS:=AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal CaltoolClient CustomLoggingClient SchedulerTextClient MeteoClient

# Misc do not depend from one another, compile in parallel

COMMON_MISC:=Plotter KStars Scripts getTemplateForTests PMUpdate InjectCommand UserTools

# Same as servers and clients

COMMON_SIMULATORS:= TCPGenericProtocolSim ReceiverBoardSim



COMMON_MODULES:=$(COMMON_ERRORS) $(COMMON_INTERFACES) $(COMMON_LIBRARIES) $(COMMON_SERVERS) $(COMMON_CLIENTS) $(COMMON_MISC) $(COMMON_SIMULATORS)



# These 3 following groups contain the telescope-specific code (interfaces, libraris, servers, clients, etc).

# Whenever there are no dependencies inside the same group, we try to compile these modules in parallel with the common ones.

# Refer to the SRT ifeq section below to see what we group together and whatnot.

SRT_DOC:=SRTDox

SRT_ERRORS:=

SRT_INTERFACES:=SRTAntennaInterface SRTActiveSurfaceInterface SRTMinorServoInterface SRTReceiversInterface
@@ -58,24 +76,9 @@ NT_CLIENTS:=NotoMountTextClient NotoActiveSurfaceGUIClient 
NT_MISC:=NotoScripts



ifeq ($(STATION),SRT)

	#Add all modules included in the SRT system  in this sequence

	#Doc, (Errors, Interfacies, Libraries, Servers, Clients, Misc)

	MODULES:=SRTDox \

		ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors \

		ActiveSurfaceErrors \

		CommonInterface ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface \

		SRTAntennaInterface SRTActiveSurfaceInterface SRTWeatherStationInterface SRTReceiversInterface \

		SlaLibrary IRALibrary TextWindowLibrary ParserLibrary XarcosLibrary SRTMinorServoLibrary ComponentProxy ModbusChannel PyTestingLibrary \

		AntennaBoss Observatory OTF PointingModel Refraction SkySource Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool TotalPower CustomLogger \

		SRTMount SRTActiveSurfaceBoss SRTMinorServo SRTKBandMFReceiver SRTWeatherStation SRT7GHzReceiver SRTLPBandReceiver \

		SRT5GHzReceiver PyDewarPositioner \

		SRTPyIFDistributor AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal CaltoolClient CustomLoggingClient SchedulerTextClient \

		SRTActiveSurfaceGUIClient SRTMountTextClient MinorServoBossTextClient \

		Plotter KStars SRTScripts



	################################## man related section ##################################

	IDL_DOC:=Managment Antenna Backends SRTActiveSurface WeatherStation

	CPP_DOC:=Libraries AntennaImplementation SRTActiveSurfaceImplementation BackendsImplementation ReceiversImplementation WeatherStationImplementation MinorServoImplementation



	Managment_MODULES:=ManagmentInterface

	Antenna_MODULES:=AntennaInterface SRTAntennaInterface

	Backends_MODULES:=BackendsInterface
@@ -91,29 +94,27 @@ ifeq ($(STATION),SRT) 
	WeatherStationImplementation_MODULES:=SRTWeatherStation

	SRTActiveSurfaceImplementation_MODULES:=SRTActiveSurfaceBoss

	MinorServoImplementation_MODULES:=SRTMinorServo

	

	CDB_SVN_LOCATION="SRT/Configuration/CDB"

	SYSTEM_SVN_MODULES=Common SRT



	TELESCOPE_MODULES:=$(SRT_ERRORS) $(SRT_INTERFACES) $(SRT_LIBRARIES) $(SRT_SERVERS) $(SRT_CLIENTS) $(SRT_MISC)

	#########################################################################################

	################################# build groups section ##################################

	# Common and telescope errors can be compiled in parallel

	ERRORS:=$(COMMON_ERRORS) $(SRT_ERRORS)

	# The telescope interfaces must be compiled after the common ones

	TELESCOPE_INTERFACES:=$(SRT_INTERFACES)

	# Common and telescope libraries can be compiled in parallel, after the base common sequentials libraries

	LIBRARIES:=$(COMMON_LIBRARIES) $(SRT_LIBRARIES)

	# Telescope servers might depend on other common servers (i.e. the SRTKBandMFReceiver which depends on the MFKBandBaseReceiver or AS bosses which depend on USDs and LANs implementations)

	TELESCOPE_SERVERS:=$(SRT_SERVERS)

	# Common and telescope clients can be compiled in parallel

	CLIENTS:=$(COMMON_CLIENTS) $(SRT_CLIENTS)

	# Misc are independend from one another, therefore we compile common and telescope misc in parallel

	MISC:=$(COMMON_MISC) $(SRT_MISC)

	# Same as above

	SIMULATORS:=$(COMMON_SIMULATORS)

	#########################################################################################

else ifeq ($(STATION),Medicina)

	#Add all modules included in the Medicina system in this sequence

	#Doc, (Errors, Interfacies, Libraries, Servers, Clients, Misc)

	MODULES:=MEDDox \

		ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors XBackendErrors \

		CommonInterface ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface XBackendInterface \

		MedicinaAntennaInterface MedicinaWeatherStationInterface \

		SlaLibrary IRALibrary TextWindowLibrary ParserLibrary ComponentProxy ModbusChannel XarcosLibrary \

		PyTestingLibrary MedicinaVertexLibrary \

		AntennaBoss Observatory OTF PointingModel Refraction SkySource Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool \

		TotalPower MedicinaMount MedicinaPyLocalOscillator MedicinaPyDMed CustomLogger MedicinaMinorServo MedicinaActiveSurfaceBoss \

		AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal CaltoolClient SchedulerTextClient MinorServoBossTextClient \

		MedicinaActiveSurfaceGUIClient MedicinaMountTextClient CustomLoggingClient\

		Plotter KStars Scripts MedScripts XBackend



	################################## man related section ##################################

	IDL_DOC:=Managment Antenna Backends WeatherStation Receivers

	CPP_DOC:=Libraries AntennaImplementation ManagementImplementation BackendsImplementation WeatherStationImplementation



	Managment_MODULES:=ManagmentInterface

	Antenna_MODULES:=AntennaInterface MedicinaAntennaInterface

	Backends_MODULES:=BackendsInterface XBackendInterface
@@ -126,26 +127,20 @@ else ifeq ($(STATION),Medicina) 
	ReceiversImplementation_MODULES:=ReceiversBoss

	WeatherStationImplementation_MODULES:=MedWeatherStation Metrology

	MedicinaActiveSurfaceImplementation_MODULES:=MedicinaActiveSurfaceBoss

	

	CDB_SVN_LOCATION="Medicina/Configuration/CDB"

	SYSTEM_SVN_MODULES=Common Medicina



	TELESCOPE_MODULES:=$(MED_ERRORS) $(MED_INTERFACES) $(MED_LIBRARIES) $(MED_SERVERS) $(MED_CLIENTS) $(MED_MISC)

	#########################################################################################

	################################# build groups section ##################################

	ERRORS:=$(COMMON_ERRORS) $(MED_ERRORS)

	TELESCOPE_INTERFACES:=$(MED_INTERFACES)

	LIBRARIES:=$(COMMON_LIBRARIES) $(MED_LIBRARIES)

	TELESCOPE_SERVERS:=$(MED_SERVERS)

	CLIENTS:=$(COMMON_CLIENTS) $(MED_CLIENTS)

	MISC:=$(COMMON_MISC) $(MED_MISC)

	SIMULATORS:=$(COMMON_SIMULATORS)

	#########################################################################################

else ifeq ($(STATION),Noto)

	#Add all modules included in the Noto system in this sequence

	#Doc, (Errors, Interfacies, Libraries, Servers, Clients, Misc)

	MODULES:= ActiveSurfaceErrors ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors \

		ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface \

		NotoAntennaInterface \

		SlaLibrary IRALibrary TextWindowLibrary ParserLibrary PyTestingLibrary \

		AntennaBoss Observatory OTF PointingModel Refraction SkySource Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool \

		TotalPower NotoMount CustomLogger NotoMinorServo NotoActiveSurfaceBoss \

		AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal SchedulerTextClient MinorServoBossTextClient \

		NotoMountTextClient CustomLoggingClient Scripts



	################################## man related section ##################################

	IDL_DOC:=Managment Antenna Backends Metrology

	CPP_DOC:=Libraries AntennaImplementation ManagementImplementation BackendsImplementation ReceiversImplementation



	Managment_MODULES:=ManagmentInterface

	Antenna_MODULES:=AntennaInterface NotoAntennaInterface

	Backends_MODULES:=BackendsInterface
@@ -156,11 +151,16 @@ else ifeq ($(STATION),Noto) 
	BackendsImplementation_MODULES:=TotalPower

	ReceiversImplementation_MODULES:=ReceiversBoss

	NotoActiveSurfaceImplementation_MODULES:=NotoActiveSurfaceBoss

	

	CDB_SVN_LOCATION="Noto/Configuration/CDB"

	SYSTEM_SVN_MODULES=Common Medicina Noto



	TELESCOPE_MODULES:=$(NT_ERRORS) $(NT_INTERFACES) $(NT_LIBRARIES) $(NT_SERVERS) $(NT_CLIENTS) $(NT_MISC)

	#########################################################################################

	################################# build groups section ##################################

	ERRORS:=$(COMMON_ERRORS) $(NT_ERRORS)

	TELESCOPE_INTERFACES:=$(NT_INTERFACES)

	LIBRARIES:=$(COMMON_LIBRARIES) $(NT_LIBRARIES)

	TELESCOPE_SERVERS:=$(NT_SERVERS)

	CLIENTS:=$(COMMON_CLIENTS) $(NT_CLIENTS)

	MISC:=$(COMMON_MISC) $(NT_MISC)

	SIMULATORS:=$(COMMON_SIMULATORS)

	#########################################################################################

else

	RESULT:=$(error STATION variable is not set correctly!)	

endif
@@ -170,7 +170,9 @@ endif 
CURRENT_DIR:=$(shell pwd)/..



define PathFinder

	$(shell for MOD in $(1); do find ${CURRENT_DIR} -name $${MOD} -not -path *ntroot* -not -path *CDB* -not -path *site-packages* \( -type d -o -type l \) -print; done)

	$(shell for MOD in $(if $(filter undefined,$(origin $(1))),$(1),$($(1))); \

		do find ${CURRENT_DIR}/Common ${CURRENT_DIR}/${STATION} -name $${MOD} -not -path *ntroot* -not -path *CDB* -not -path *site-packages* \( -type d -o -type l \) -print 2>/dev/null; \

	done)

endef



#if the INTROOT varaible is not defined the make is stopped
@@ -183,21 +185,6 @@ ifndef ACS_CDB 
	RESULT:=$(error ACS_CDB variable is not set!)

endif



# #if the SYSLOCATION variable is not set the make process is stopped

# ifndef SYSLOCATION

# 	RESULT:=$(error SYSLOCATION variable is not set!)

# endif



# #if the SVNUSER variable is not set the make process is stopped

# ifndef SVNUSER

# 	RESULT:=$(error SVNUSER variable is not set!)

# endif



# #if the SYSTEM_SVN_TAG variable is not set the make process is stopped

# ifndef SYSTEM_SVN_TAG

# 	RESULT:=$(error SYSTEM_SVN_TAG variable is not set!)

# endif



#set up the Modules file for each IDL and CPP documentation

$(foreach i, $(IDL_DOC), $(eval $(i)_MODULES_PATH:=$(call PathFinder,$($(i)_MODULES))))

$(foreach i, $(CPP_DOC), $(eval $(i)_MODULES_PATH:=$(call PathFinder,$($(i)_MODULES))))
@@ -205,76 +192,81 @@ $(foreach i, $(CPP_DOC), $(eval $(i)_MODULES_PATH:=$(call PathFinder,$($(i)_MODU 
$(foreach i, $(CPP_DOC), $(shell echo "INPUT=$($(i)_MODULES_PATH)" > /tmp/$(i)_cpp.in))

$(foreach i, $(IDL_DOC), $(shell echo "INPUT=$($(i)_MODULES_PATH)" > /tmp/$(i)_idl.in))



COMPILE_MODULES_PATH:=$(call PathFinder,$(MODULES))

BUILD_DIRS:=$(foreach i, $(COMPILE_MODULES_PATH), $(i)/src)



COMMON_MODULES_PATH:=$(call PathFinder,$(COMMON_MODULES))

COMMON_MODULES_DIRS:=$(foreach i, $(COMMON_MODULES_PATH), $(i)/src)

COMMON_BUILD_TARGETS:=$(COMMON_MODULES_DIRS:%=build-%)

COMMON_CLEAN_TARGETS:=$(COMMON_MODULES_DIRS:%=clean-%)

TELESCOPE_MODULES_PATH:=$(call PathFinder,$(TELESCOPE_MODULES))

TELESCOPE_MODULES_DIRS:=$(foreach i, $(TELESCOPE_MODULES_PATH), $(i)/src)

TELESCOPE_BUILD_TARGETS:=$(TELESCOPE_MODULES_DIRS:%=build-%)

TELESCOPE_CLEAN_TARGETS:=$(TELESCOPE_MODULES_DIRS:%=clean-%)



ifeq (X$(DOCROOT),X)

	DOC:=$(HOME)/docroot

else

	DOC:=$(DOCROOT)

endif





DOCIDL:=$(DOC)/idl

DOCCPP:=$(DOC)/cpp



.PHONY: all

all: common-build telescope-build



.PHONY: clean

clean: common-clean telescope-clean



.PHONY: common-build $(COMMON_BUILD_TARGETS)

common-build:

	@for i in $(COMMON_MODULES_DIRS); do \

		$(MAKE) -C $${i} all; \

		$(MAKE) -C $${i} install; \

	done



.PHONY: common-clean $(COMMON_CLEAN_TARGETS)

common-clean: $(COMMON_CLEAN_TARGETS)

$(COMMON_CLEAN_TARGETS):

	$(MAKE) -C $(@:clean-%=%) clean_dist



.PHONY: telescope-build $(TELESCOPE_BUILD_TARGETS)

telescope-build: common-build

	@for i in $(TELESCOPE_MODULES_DIRS); do \

		$(MAKE) -C $${i} all; \

		$(MAKE) -C $${i} install; \

	done



.PHONY: telescope-clean $(TELESCOPE_CLEAN_TARGETS)

telescope-clean: $(TELESCOPE_CLEAN_TARGETS)

$(TELESCOPE_CLEAN_TARGETS):

	$(MAKE) -C $(@:clean-%=%) clean_dist

# Sequential targets like COMMON_SEQ_LIBS have to be expanded in order to avoid unwanted parallelization.

# This order should not be changed in order for the build process to succeed. Simulators are disabled by default since we mostly rely on the discos-simulator suite.

TARGETS:=ERRORS BASE_INTERFACES CORE_INTERFACES EXTENDED_INTERFACES TELESCOPE_INTERFACES $(COMMON_SEQ_LIBS) LIBRARIES COMMON_SERVERS TELESCOPE_SERVERS CLIENTS MISC #SIMULATORS



# This function is used to dynamically declare the build and clean targets

define DeclareTarget

$1_DIRS:=$(addsuffix /src, $(call PathFinder,$1))

$1_BUILD_TARGETS:=$$($1_DIRS:%=build-%)

$1_CLEAN_TARGETS:=$$($1_DIRS:%=clean-%)



.PHONY: build-$1 $$($1_BUILD_TARGETS)

build-$1: $$($1_BUILD_TARGETS)

$$($1_BUILD_TARGETS):

	+$(MAKE) -C $$(@:build-%=%) all

	+$(MAKE) -C $$(@:build-%=%) install



.PHONY: clean-$1 $$($1_CLEAN_TARGETS)

clean-$1: $$($1_CLEAN_TARGETS)

$$($1_CLEAN_TARGETS):

	+$(MAKE) -C $$(@:clean-%=%) clean_dist

endef



.PHONY: compile

compile:

	@for i in $(BUILD_DIRS); do \

		$(MAKE) -C $${i} all ; \

# Main target, builds and installs all. Suggested to be launched with "make -j" in order to speed up the build process considerably.

.PHONY: all $(addprefix build-,$(TARGETS))

all:

	@for i in $(TARGETS); do \

		$(MAKE) build-$${i}; \

	done



.PHONY: install

install:

	@for i in $(BUILD_DIRS); do \

		$(MAKE) -C $${i} install ; \

	done

# The following line dynamically declares all the make targets starting from the TARGETS variable

$(foreach t, $(TARGETS), $(eval $(call DeclareTarget,$(t))))



# This target removes all the .gitignore ignored files from the repository, effectively removing all the object, lib and bin directories, etc.

# It only works if discos was cloned via git

.PHONY: clean-repo

clean-repo:

	@echo -n "Cleaning the repository from ignored files..."

	-@git clean -fdX ..

	@echo "done!"



# This target cleans all the INTROOT so that a clean build process can be executed again

.PHONY: clean-dist

clean-dist:

	@echo -n "Cleaning INTROOT..."

	@rm -rf $(INTROOT)/app-defaults/*

	@rm -rf $(INTROOT)/bin/*

	@rm -rf $(INTROOT)/config/CDB/schemas/*

	@rm -rf $(INTROOT)/idl/*

	@rm -rf $(INTROOT)/include/*

	@rm -rf $(INTROOT)/lib/ACScomponents/*

	@rm -rf $(INTROOT)/lib/endorsed/*

	@find $(INTROOT)/lib/python -mindepth 1 ! -name 'site-packages' -exec rm -rf {} +

	@rm -rf $(INTROOT)/lib/python/site-packages/*

	@rm -rf $(INTROOT)/man/man*/*

	@rm -rf $(INTROOT)/Sources/*

	@rm -rf $(INTROOT)/templates/*

	@rm -rf $(INTROOT)/user/bin/*

	@echo "done!"



# cdb:

# 	@cd $(ACS_CDB); ln -s $(SYSLOCATION)/$(SYSTEM_SVN_TAG)/$(CDB_SVN_LOCATION) CDB

# 

# sources:

# 	@cd $(SYSLOCATION) ; svn checkout svn+ssh://$(SVNUSER)@nuraghe-devel.oa-cagliari.inaf.it/ACS/tags/$(SYSTEM_SVN_TAG)/$${i}

################################### DOC related targets #################################

.PHONY: stat

stat:

	@./slocc.sh -findopt "( -name *.cpp -o -name *.i -o -name *.h -o -name *.idl -o -name *.xml -o -name *.xsd -o -name *.py -o -name Makefile* ) \

	-not -path *object* -not -path *lib*" ..



.PHONY: man

man:

	@echo "creating documentation tree....."

	@if [ ! -d $(DOC) ]; then mkdir $(DOC) ; fi
@@ -302,25 +294,8 @@ man: 
		rm /tmp/doxyconf ; \

		rm /tmp/$${i}_cpp.in ; \

	done



	@echo "documentation done in $(DOC)"



legacy-all:

	@for i in $(BUILD_DIRS); do \

		make -C $${i} all install ; \

	done



legacy-clean:

	@rm -rf $(INTROOT)/bin/*

	@rm -rf $(INTROOT)/config/CDB/schemas/*

	@rm -rf $(INTROOT)/idl/*

	@rm -rf $(INTROOT)/Sources/*

	@rm -rf $(INTROOT)/templates/*

	@rm -rf $(INTROOT)/includes/*

	@for i in $(BUILD_DIRS); do \

		make -C $${i} clean; \

	done

.PHONY: clean-doc

clean-doc:

	@rm -rf $(DOC)/*
 
stat:

	@./slocc.sh -findopt "( -name *.cpp -o -name *.i -o -name *.h -o -name *.idl -o -name *.xml -o -name *.xsd -o -name *.py -o -name Makefile* ) \

	-not -path *object* -not -path *lib*" ..