From: Gleb Mazovetskiy <glex.spb@gmail.com>
Date: Sun, 3 Jan 2021 17:15:25 +0000 (+0000)
Subject: Makefile: Parallelize glibc locale generation
X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=6fbdf51596079b17ef360e942b604237efbcc217;p=buildroot.git

Makefile: Parallelize glibc locale generation

Parallelizes locale generation based on `BR2_JLEVEL` setting.

Locale generation always runs during the finalize stage and can consume
a significant amount of time. Parallelizing it greatly reduces that time
on multi-core machines.

To parallelize it, we first invoke `localedef` for every locale in
parallel with the `--no-archive` option. This creates the intermediate
locale data instead of writing to the finally archive directly.

Then, we invoke `localedef` again once to create the archive from the
intermediate compiled locale data files.

We have to do it this way because `localedef` does not do any locking
when writing to the archive file, so calling it without `--no-archive`
concurrently could result in a corrupt archive file or an archive file
that is missing some locales.

While we're at it, make two additional improvements:
- Remove locale-archive before adding to it. Otherwise, repeated
  applications of target-finalize will keep on growing the file.
- Sort the locales when creating locale-archive so its contents are
  reproducible.

We use `find` to collect the installed locales rather than LOCALES. This
makes it possible for something else (skeleton, overlay, custom package)
to create and install additional locales and still have them added to
locale-archive.

Signed-off-by: Gleb Mazovetskiy <glex.spb@gmail.com>
[Arnout:
 - Remove -j$(PARALLEL_JOBS), it's already part of $(MAKE)
 - Remove HOST_DIR, TARGET_DIR, STAGING_DIR, they're already exported
 - Extend commit message
]
Signed-off-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>
---

diff --git a/Makefile b/Makefile
index 74c238cb20..d3c67c20d4 100644
--- a/Makefile
+++ b/Makefile
@@ -663,32 +663,17 @@ endef
 TARGET_FINALIZE_HOOKS += TOOLCHAIN_ECLIPSE_REGISTER
 endif
 
-# Generate locale data. Basically, we call the localedef program
-# (built by the host-localedef package) for each locale. The input
-# data comes preferably from the toolchain, or if the toolchain does
-# not have them (Linaro toolchains), we use the ones available on the
-# host machine.
+# Generate locale data.
 ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),y)
 GLIBC_GENERATE_LOCALES = $(call qstrip,$(BR2_GENERATE_LOCALE))
 ifneq ($(GLIBC_GENERATE_LOCALES),)
 PACKAGES += host-localedef
 
 define GENERATE_GLIBC_LOCALES
-	$(Q)mkdir -p $(TARGET_DIR)/usr/lib/locale/
-	$(Q)for locale in $(GLIBC_GENERATE_LOCALES) ; do \
-		inputfile=`echo $${locale} | cut -f1 -d'.'` ; \
-		charmap=`echo $${locale} | cut -f2 -d'.' -s` ; \
-		if test -z "$${charmap}" ; then \
-			charmap="UTF-8" ; \
-		fi ; \
-		echo "Generating locale $${inputfile}.$${charmap}" ; \
-		I18NPATH=$(STAGING_DIR)/usr/share/i18n:/usr/share/i18n \
-		$(HOST_DIR)/bin/localedef \
-			--prefix=$(TARGET_DIR) \
-			--$(call LOWERCASE,$(BR2_ENDIAN))-endian \
-			-i $${inputfile} -f $${charmap} \
-			$${locale} ; \
-	done
+	$(MAKE) -f support/misc/gen-glibc-locales.mk \
+		ENDIAN=$(call LOWERCASE,$(BR2_ENDIAN)) \
+		LOCALES="$(GLIBC_GENERATE_LOCALES)" \
+		Q=$(Q)
 endef
 TARGET_FINALIZE_HOOKS += GENERATE_GLIBC_LOCALES
 endif
diff --git a/support/misc/gen-glibc-locales.mk b/support/misc/gen-glibc-locales.mk
new file mode 100644
index 0000000000..3db1e0dc11
--- /dev/null
+++ b/support/misc/gen-glibc-locales.mk
@@ -0,0 +1,41 @@
+# Generates glibc locale data for target.
+
+inputfile = $(firstword $(subst ., ,$(1)))
+charmap = $(or $(word 2,$(subst ., ,$(1))),UTF-8)
+
+# Packages all the generated locale data into the final archive.
+#
+# We sort the file names to produce consistent output regardless of
+# the `find` outputs order.
+$(TARGET_DIR)/usr/lib/locale/locale-archive: $(LOCALES)
+	$(Q)rm -f $(@)
+	$(Q)find $(TARGET_DIR)/usr/lib/locale/ -maxdepth 1 -mindepth 1 -type d -print0 \
+	| sort -z \
+	| xargs -0 \
+		$(HOST_DIR)/bin/localedef \
+			--prefix=$(TARGET_DIR) \
+			--$(ENDIAN)-endian \
+			--add-to-archive
+
+# Generates locale data for each locale.
+#
+# The input data comes preferably from the toolchain, or if the toolchain
+# does not have them (Linaro toolchains), we use the ones available on the
+# host machine.
+#
+# Uses `localedef`, which is built by the `host-localedef` package.
+$(LOCALES): | $(TARGET_DIR)/usr/lib/locale/
+	$(Q)echo "Generating locale $(@)"
+	$(Q)I18NPATH=$(STAGING_DIR)/usr/share/i18n:/usr/share/i18n \
+	$(HOST_DIR)/bin/localedef \
+		--prefix=$(TARGET_DIR) \
+		--$(ENDIAN)-endian \
+		--no-archive \
+		-i $(call inputfile,$(@)) \
+		-f $(call charmap,$(@)) \
+		$(@)
+
+.PHONY: $(LOCALES)
+
+$(TARGET_DIR)/usr/lib/locale/:
+	$(Q)mkdir -p $(TARGET_DIR)/usr/lib/locale/