Commit 5491a0f3 authored by Giuseppe Carboni's avatar Giuseppe Carboni
Browse files

Major Makefile rework

For more information check the Makefile for useful comments about the major refactoring.
With a 12 cores virtual machine, compiling DISCOS with this Makefile using `time make -j` result in less than 14 minutes of real time. Quite a big leap from the 40+ minutes that a sequential build takes.
parent 2478ea9e
Loading
Loading
Loading
Loading
+130 −155
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@
# Bartolini 02/02/15  added .PHONY target tall
# Bartolini 03/02/15  split away common, inspired by http://lackof.org/taggart/hacking/make-example/
# Bartolini 25/08/15  Added comments and changed default target for all
# Carboni   14/04/25  Enabled correct handling of parallel jobs (-j flag)
# Carboni   20/04/25  Parallelized code compilation with make -j

#
# this new Makefile section separates code modules between what is common
@@ -21,18 +21,36 @@
# and there is no need to replicate common code modules for all telescopes.
#

# Documentation under Common directory
COMMON_DOC:=
# Common errors, can all be compiled in parallel
COMMON_ERRORS:=ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors XBackendErrors ActiveSurfaceErrors
COMMON_INTERFACES:=CommonInterface ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface XBackendInterface
COMMON_LIBRARIES:=SlaLibrary IRALibrary DiscosVersion TextWindowLibrary ParserLibrary XarcosLibrary ModbusChannel ComponentProxy DiscosLocals XEphemAstroLib DiscosBackendProtocol PyTestingLibrary
# The CommonInterface is the only interface that has no other dependency, must be compiled first
BASE_INTERFACES:=CommonInterface
# The following core interfaces has no dependency between them, they only depend on CommonInterface, therefore they can be compiled in parallel
CORE_INTERFACES:=ManagmentInterface AntennaInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface
# These common interfaces depends on some interface listed in the line above, must be compiled separately
EXTENDED_INTERFACES:=ReceiversInterface XBackendInterface
# These libraries must be compiled sequentially, since the DiscosVersion library depends on the IRALibrary which depends on the SlaLibrary
COMMON_SEQ_LIBS:=SlaLibrary IRALibrary DiscosVersion
# All these other libraries only depend on the libraries listed above, therefore they can be compiled in parallel
COMMON_LIBRARIES:=TextWindowLibrary ParserLibrary XarcosLibrary ModbusChannel ComponentProxy DiscosLocals XEphemAstroLib DiscosBackendProtocol PyTestingLibrary
# All the common servers are independend from one another, they only depend on interfaces, therefore we can compile them in parallel.
# There may be some hidden dependency between servers, but it may never come up since each core compiles a single cpp file.
# Issues may only arise if we use a very high number of cores, which is unlikely.
# If we'll ever face this problem we will split the SERVERS into more groups, with the latter one having servers which depends on some other servers in the first group.
COMMON_SERVERS:=AntennaBoss Observatory OTF PointingModel Refraction SkySource SolarSystem Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool TotalPower CustomLogger \
				PyDewarPositioner Sardara Skarab PyLocalOscillator MFKBandBaseReceiver PyCalmux ActiveSurfaceLan ActiveSurfaceUSD XBackend
# Same as servers
COMMON_CLIENTS:=AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal CaltoolClient CustomLoggingClient SchedulerTextClient MeteoClient
# Misc do not depend from one another, compile in parallel
COMMON_MISC:=Plotter KStars Scripts getTemplateForTests PMUpdate InjectCommand UserTools
# Same as servers and clients
COMMON_SIMULATORS:= TCPGenericProtocolSim ReceiverBoardSim

COMMON_MODULES:=$(COMMON_ERRORS) $(COMMON_INTERFACES) $(COMMON_LIBRARIES) $(COMMON_SERVERS) $(COMMON_CLIENTS) $(COMMON_MISC) $(COMMON_SIMULATORS)

# These 3 following groups contain the telescope-specific code (interfaces, libraris, servers, clients, etc).
# Whenever there are no dependencies inside the same group, we try to compile these modules in parallel with the common ones.
# Refer to the SRT ifeq section below to see what we group together and whatnot.
SRT_DOC:=SRTDox
SRT_ERRORS:=
SRT_INTERFACES:=SRTAntennaInterface SRTActiveSurfaceInterface SRTMinorServoInterface SRTReceiversInterface
@@ -58,24 +76,9 @@ NT_CLIENTS:=NotoMountTextClient NotoActiveSurfaceGUIClient
NT_MISC:=NotoScripts

ifeq ($(STATION),SRT)
	#Add all modules included in the SRT system  in this sequence
	#Doc, (Errors, Interfacies, Libraries, Servers, Clients, Misc)
	MODULES:=SRTDox \
		ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors \
		ActiveSurfaceErrors \
		CommonInterface ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface \
		SRTAntennaInterface SRTActiveSurfaceInterface SRTWeatherStationInterface SRTReceiversInterface \
		SlaLibrary IRALibrary TextWindowLibrary ParserLibrary XarcosLibrary SRTMinorServoLibrary ComponentProxy ModbusChannel PyTestingLibrary \
		AntennaBoss Observatory OTF PointingModel Refraction SkySource Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool TotalPower CustomLogger \
		SRTMount SRTActiveSurfaceBoss SRTMinorServo SRTKBandMFReceiver SRTWeatherStation SRT7GHzReceiver SRTLPBandReceiver \
		SRT5GHzReceiver PyDewarPositioner \
		SRTPyIFDistributor AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal CaltoolClient CustomLoggingClient SchedulerTextClient \
		SRTActiveSurfaceGUIClient SRTMountTextClient MinorServoBossTextClient \
		Plotter KStars SRTScripts

	################################## man related section ##################################
	IDL_DOC:=Managment Antenna Backends SRTActiveSurface WeatherStation
	CPP_DOC:=Libraries AntennaImplementation SRTActiveSurfaceImplementation BackendsImplementation ReceiversImplementation WeatherStationImplementation MinorServoImplementation

	Managment_MODULES:=ManagmentInterface
	Antenna_MODULES:=AntennaInterface SRTAntennaInterface
	Backends_MODULES:=BackendsInterface
@@ -91,29 +94,27 @@ ifeq ($(STATION),SRT)
	WeatherStationImplementation_MODULES:=SRTWeatherStation
	SRTActiveSurfaceImplementation_MODULES:=SRTActiveSurfaceBoss
	MinorServoImplementation_MODULES:=SRTMinorServo
	
	CDB_SVN_LOCATION="SRT/Configuration/CDB"
	SYSTEM_SVN_MODULES=Common SRT

	TELESCOPE_MODULES:=$(SRT_ERRORS) $(SRT_INTERFACES) $(SRT_LIBRARIES) $(SRT_SERVERS) $(SRT_CLIENTS) $(SRT_MISC)
	#########################################################################################
	################################# build groups section ##################################
	# Common and telescope errors can be compiled in parallel
	ERRORS:=$(COMMON_ERRORS) $(SRT_ERRORS)
	# The telescope interfaces must be compiled after the common ones
	TELESCOPE_INTERFACES:=$(SRT_INTERFACES)
	# Common and telescope libraries can be compiled in parallel, after the base common sequentials libraries
	LIBRARIES:=$(COMMON_LIBRARIES) $(SRT_LIBRARIES)
	# Telescope servers might depend on other common servers (i.e. the SRTKBandMFReceiver which depends on the MFKBandBaseReceiver or AS bosses which depend on USDs and LANs implementations)
	TELESCOPE_SERVERS:=$(SRT_SERVERS)
	# Common and telescope clients can be compiled in parallel
	CLIENTS:=$(COMMON_CLIENTS) $(SRT_CLIENTS)
	# Misc are independend from one another, therefore we compile common and telescope misc in parallel
	MISC:=$(COMMON_MISC) $(SRT_MISC)
	# Same as above
	SIMULATORS:=$(COMMON_SIMULATORS)
	#########################################################################################
else ifeq ($(STATION),Medicina)
	#Add all modules included in the Medicina system in this sequence
	#Doc, (Errors, Interfacies, Libraries, Servers, Clients, Misc)
	MODULES:=MEDDox \
		ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors XBackendErrors \
		CommonInterface ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface XBackendInterface \
		MedicinaAntennaInterface MedicinaWeatherStationInterface \
		SlaLibrary IRALibrary TextWindowLibrary ParserLibrary ComponentProxy ModbusChannel XarcosLibrary \
		PyTestingLibrary MedicinaVertexLibrary \
		AntennaBoss Observatory OTF PointingModel Refraction SkySource Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool \
		TotalPower MedicinaMount MedicinaPyLocalOscillator MedicinaPyDMed CustomLogger MedicinaMinorServo MedicinaActiveSurfaceBoss \
		AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal CaltoolClient SchedulerTextClient MinorServoBossTextClient \
		MedicinaActiveSurfaceGUIClient MedicinaMountTextClient CustomLoggingClient\
		Plotter KStars Scripts MedScripts XBackend

	################################## man related section ##################################
	IDL_DOC:=Managment Antenna Backends WeatherStation Receivers
	CPP_DOC:=Libraries AntennaImplementation ManagementImplementation BackendsImplementation WeatherStationImplementation

	Managment_MODULES:=ManagmentInterface
	Antenna_MODULES:=AntennaInterface MedicinaAntennaInterface
	Backends_MODULES:=BackendsInterface XBackendInterface
@@ -126,26 +127,20 @@ else ifeq ($(STATION),Medicina)
	ReceiversImplementation_MODULES:=ReceiversBoss
	WeatherStationImplementation_MODULES:=MedWeatherStation Metrology
	MedicinaActiveSurfaceImplementation_MODULES:=MedicinaActiveSurfaceBoss
	
	CDB_SVN_LOCATION="Medicina/Configuration/CDB"
	SYSTEM_SVN_MODULES=Common Medicina

	TELESCOPE_MODULES:=$(MED_ERRORS) $(MED_INTERFACES) $(MED_LIBRARIES) $(MED_SERVERS) $(MED_CLIENTS) $(MED_MISC)
	#########################################################################################
	################################# build groups section ##################################
	ERRORS:=$(COMMON_ERRORS) $(MED_ERRORS)
	TELESCOPE_INTERFACES:=$(MED_INTERFACES)
	LIBRARIES:=$(COMMON_LIBRARIES) $(MED_LIBRARIES)
	TELESCOPE_SERVERS:=$(MED_SERVERS)
	CLIENTS:=$(COMMON_CLIENTS) $(MED_CLIENTS)
	MISC:=$(COMMON_MISC) $(MED_MISC)
	SIMULATORS:=$(COMMON_SIMULATORS)
	#########################################################################################
else ifeq ($(STATION),Noto)
	#Add all modules included in the Noto system in this sequence
	#Doc, (Errors, Interfacies, Libraries, Servers, Clients, Misc)
	MODULES:= ActiveSurfaceErrors ClientErrors ComponentErrors AntennaErrors ParserErrors BackendsErrors ManagementErrors ReceiversErrors MetrologyErrors MinorServoErrors \
		ManagmentInterface AntennaInterface ReceiversInterface BackendsInterface MinorServoInterface WeatherStationInterface ActiveSurfaceInterface \
		NotoAntennaInterface \
		SlaLibrary IRALibrary TextWindowLibrary ParserLibrary PyTestingLibrary \
		AntennaBoss Observatory OTF PointingModel Refraction SkySource Moon FitsWriter Scheduler ReceiversBoss ExternalClients CalibrationTool \
		TotalPower NotoMount CustomLogger NotoMinorServo NotoActiveSurfaceBoss \
		AntennaBossTextClient ObservatoryTextClient GenericBackendTextClient ReceiversBossTextClient SystemTerminal SchedulerTextClient MinorServoBossTextClient \
		NotoMountTextClient CustomLoggingClient Scripts

	################################## man related section ##################################
	IDL_DOC:=Managment Antenna Backends Metrology
	CPP_DOC:=Libraries AntennaImplementation ManagementImplementation BackendsImplementation ReceiversImplementation

	Managment_MODULES:=ManagmentInterface
	Antenna_MODULES:=AntennaInterface NotoAntennaInterface
	Backends_MODULES:=BackendsInterface
@@ -156,11 +151,16 @@ else ifeq ($(STATION),Noto)
	BackendsImplementation_MODULES:=TotalPower
	ReceiversImplementation_MODULES:=ReceiversBoss
	NotoActiveSurfaceImplementation_MODULES:=NotoActiveSurfaceBoss
	
	CDB_SVN_LOCATION="Noto/Configuration/CDB"
	SYSTEM_SVN_MODULES=Common Medicina Noto

	TELESCOPE_MODULES:=$(NT_ERRORS) $(NT_INTERFACES) $(NT_LIBRARIES) $(NT_SERVERS) $(NT_CLIENTS) $(NT_MISC)
	#########################################################################################
	################################# build groups section ##################################
	ERRORS:=$(COMMON_ERRORS) $(NT_ERRORS)
	TELESCOPE_INTERFACES:=$(NT_INTERFACES)
	LIBRARIES:=$(COMMON_LIBRARIES) $(NT_LIBRARIES)
	TELESCOPE_SERVERS:=$(NT_SERVERS)
	CLIENTS:=$(COMMON_CLIENTS) $(NT_CLIENTS)
	MISC:=$(COMMON_MISC) $(NT_MISC)
	SIMULATORS:=$(COMMON_SIMULATORS)
	#########################################################################################
else
	RESULT:=$(error STATION variable is not set correctly!)	
endif
@@ -170,7 +170,9 @@ endif
CURRENT_DIR:=$(shell pwd)/..

define PathFinder
	$(shell for MOD in $(1); do find ${CURRENT_DIR} -name $${MOD} -not -path *ntroot* -not -path *CDB* -not -path *site-packages* \( -type d -o -type l \) -print; done)
	$(shell for MOD in $(if $(filter undefined,$(origin $(1))),$(1),$($(1))); \
		do find ${CURRENT_DIR}/Common ${CURRENT_DIR}/${STATION} -name $${MOD} -not -path *ntroot* -not -path *CDB* -not -path *site-packages* \( -type d -o -type l \) -print 2>/dev/null; \
	done)
endef

#if the INTROOT varaible is not defined the make is stopped
@@ -183,21 +185,6 @@ ifndef ACS_CDB
	RESULT:=$(error ACS_CDB variable is not set!)
endif

# #if the SYSLOCATION variable is not set the make process is stopped
# ifndef SYSLOCATION
# 	RESULT:=$(error SYSLOCATION variable is not set!)
# endif

# #if the SVNUSER variable is not set the make process is stopped
# ifndef SVNUSER
# 	RESULT:=$(error SVNUSER variable is not set!)
# endif

# #if the SYSTEM_SVN_TAG variable is not set the make process is stopped
# ifndef SYSTEM_SVN_TAG
# 	RESULT:=$(error SYSTEM_SVN_TAG variable is not set!)
# endif

#set up the Modules file for each IDL and CPP documentation
$(foreach i, $(IDL_DOC), $(eval $(i)_MODULES_PATH:=$(call PathFinder,$($(i)_MODULES))))
$(foreach i, $(CPP_DOC), $(eval $(i)_MODULES_PATH:=$(call PathFinder,$($(i)_MODULES))))
@@ -205,76 +192,81 @@ $(foreach i, $(CPP_DOC), $(eval $(i)_MODULES_PATH:=$(call PathFinder,$($(i)_MODU
$(foreach i, $(CPP_DOC), $(shell echo "INPUT=$($(i)_MODULES_PATH)" > /tmp/$(i)_cpp.in))
$(foreach i, $(IDL_DOC), $(shell echo "INPUT=$($(i)_MODULES_PATH)" > /tmp/$(i)_idl.in))

COMPILE_MODULES_PATH:=$(call PathFinder,$(MODULES))
BUILD_DIRS:=$(foreach i, $(COMPILE_MODULES_PATH), $(i)/src)

COMMON_MODULES_PATH:=$(call PathFinder,$(COMMON_MODULES))
COMMON_MODULES_DIRS:=$(foreach i, $(COMMON_MODULES_PATH), $(i)/src)
COMMON_BUILD_TARGETS:=$(COMMON_MODULES_DIRS:%=build-%)
COMMON_CLEAN_TARGETS:=$(COMMON_MODULES_DIRS:%=clean-%)
TELESCOPE_MODULES_PATH:=$(call PathFinder,$(TELESCOPE_MODULES))
TELESCOPE_MODULES_DIRS:=$(foreach i, $(TELESCOPE_MODULES_PATH), $(i)/src)
TELESCOPE_BUILD_TARGETS:=$(TELESCOPE_MODULES_DIRS:%=build-%)
TELESCOPE_CLEAN_TARGETS:=$(TELESCOPE_MODULES_DIRS:%=clean-%)

ifeq (X$(DOCROOT),X)
	DOC:=$(HOME)/docroot
else
	DOC:=$(DOCROOT)
endif


DOCIDL:=$(DOC)/idl
DOCCPP:=$(DOC)/cpp

.PHONY: all
all: common-build telescope-build

.PHONY: clean
clean: common-clean telescope-clean

.PHONY: common-build $(COMMON_BUILD_TARGETS)
common-build:
	@for i in $(COMMON_MODULES_DIRS); do \
		$(MAKE) -C $${i} all; \
		$(MAKE) -C $${i} install; \
	done

.PHONY: common-clean $(COMMON_CLEAN_TARGETS)
common-clean: $(COMMON_CLEAN_TARGETS)
$(COMMON_CLEAN_TARGETS):
	$(MAKE) -C $(@:clean-%=%) clean_dist

.PHONY: telescope-build $(TELESCOPE_BUILD_TARGETS)
telescope-build: common-build
	@for i in $(TELESCOPE_MODULES_DIRS); do \
		$(MAKE) -C $${i} all; \
		$(MAKE) -C $${i} install; \
	done

.PHONY: telescope-clean $(TELESCOPE_CLEAN_TARGETS)
telescope-clean: $(TELESCOPE_CLEAN_TARGETS)
$(TELESCOPE_CLEAN_TARGETS):
	$(MAKE) -C $(@:clean-%=%) clean_dist
# Sequential targets like COMMON_SEQ_LIBS have to be expanded in order to avoid unwanted parallelization.
# This order should not be changed in order for the build process to succeed. Simulators are disabled by default since we mostly rely on the discos-simulator suite.
TARGETS:=ERRORS BASE_INTERFACES CORE_INTERFACES EXTENDED_INTERFACES TELESCOPE_INTERFACES $(COMMON_SEQ_LIBS) LIBRARIES COMMON_SERVERS TELESCOPE_SERVERS CLIENTS MISC #SIMULATORS

# This function is used to dynamically declare the build and clean targets
define DeclareTarget
$1_DIRS:=$(addsuffix /src, $(call PathFinder,$1))
$1_BUILD_TARGETS:=$$($1_DIRS:%=build-%)
$1_CLEAN_TARGETS:=$$($1_DIRS:%=clean-%)

.PHONY: build-$1 $$($1_BUILD_TARGETS)
build-$1: $$($1_BUILD_TARGETS)
$$($1_BUILD_TARGETS):
	+$(MAKE) -C $$(@:build-%=%) all
	+$(MAKE) -C $$(@:build-%=%) install

.PHONY: clean-$1 $$($1_CLEAN_TARGETS)
clean-$1: $$($1_CLEAN_TARGETS)
$$($1_CLEAN_TARGETS):
	+$(MAKE) -C $$(@:clean-%=%) clean_dist
endef

.PHONY: compile
compile:
	@for i in $(BUILD_DIRS); do \
		$(MAKE) -C $${i} all ; \
# Main target, builds and installs all. Suggested to be launched with "make -j" in order to speed up the build process considerably.
.PHONY: all $(addprefix build-,$(TARGETS))
all:
	@for i in $(TARGETS); do \
		$(MAKE) build-$${i}; \
	done

.PHONY: install
install:
	@for i in $(BUILD_DIRS); do \
		$(MAKE) -C $${i} install ; \
	done
# The following line dynamically declares all the make targets starting from the TARGETS variable
$(foreach t, $(TARGETS), $(eval $(call DeclareTarget,$(t))))

# This target removes all the .gitignore ignored files from the repository, effectively removing all the object, lib and bin directories, etc.
# It only works if discos was cloned via git
.PHONY: clean-repo
clean-repo:
	@echo -n "Cleaning the repository from ignored files..."
	-@git clean -fdX ..
	@echo "done!"

# This target cleans all the INTROOT so that a clean build process can be executed again
.PHONY: clean-dist
clean-dist:
	@echo -n "Cleaning INTROOT..."
	@rm -rf $(INTROOT)/app-defaults/*
	@rm -rf $(INTROOT)/bin/*
	@rm -rf $(INTROOT)/config/CDB/schemas/*
	@rm -rf $(INTROOT)/idl/*
	@rm -rf $(INTROOT)/include/*
	@rm -rf $(INTROOT)/lib/ACScomponents/*
	@rm -rf $(INTROOT)/lib/endorsed/*
	@find $(INTROOT)/lib/python -mindepth 1 ! -name 'site-packages' -exec rm -rf {} +
	@rm -rf $(INTROOT)/lib/python/site-packages/*
	@rm -rf $(INTROOT)/man/man*/*
	@rm -rf $(INTROOT)/Sources/*
	@rm -rf $(INTROOT)/templates/*
	@rm -rf $(INTROOT)/user/bin/*
	@echo "done!"

# cdb:
# 	@cd $(ACS_CDB); ln -s $(SYSLOCATION)/$(SYSTEM_SVN_TAG)/$(CDB_SVN_LOCATION) CDB
# 
# sources:
# 	@cd $(SYSLOCATION) ; svn checkout svn+ssh://$(SVNUSER)@nuraghe-devel.oa-cagliari.inaf.it/ACS/tags/$(SYSTEM_SVN_TAG)/$${i}
################################### DOC related targets #################################
.PHONY: stat
stat:
	@./slocc.sh -findopt "( -name *.cpp -o -name *.i -o -name *.h -o -name *.idl -o -name *.xml -o -name *.xsd -o -name *.py -o -name Makefile* ) \
	-not -path *object* -not -path *lib*" ..

.PHONY: man
man:
	@echo "creating documentation tree....."
	@if [ ! -d $(DOC) ]; then mkdir $(DOC) ; fi
@@ -302,25 +294,8 @@ man:
		rm /tmp/doxyconf ; \
		rm /tmp/$${i}_cpp.in ; \
	done

	@echo "documentation done in $(DOC)"

legacy-all:
	@for i in $(BUILD_DIRS); do \
		make -C $${i} all install ; \
	done

legacy-clean:
	@rm -rf $(INTROOT)/bin/*
	@rm -rf $(INTROOT)/config/CDB/schemas/*
	@rm -rf $(INTROOT)/idl/*
	@rm -rf $(INTROOT)/Sources/*
	@rm -rf $(INTROOT)/templates/*
	@rm -rf $(INTROOT)/includes/*
	@for i in $(BUILD_DIRS); do \
		make -C $${i} clean; \
	done
.PHONY: clean-doc
clean-doc:
	@rm -rf $(DOC)/*
stat:
	@./slocc.sh -findopt "( -name *.cpp -o -name *.i -o -name *.h -o -name *.idl -o -name *.xml -o -name *.xsd -o -name *.py -o -name Makefile* ) \
	-not -path *object* -not -path *lib*" ..