Refactor xmllint check and make it fail on failure

This commit also adds a check to see that all files that
are part of an xi:include also have part of XML_FILES
and vice versa. It also fixes any applications where this
was not true.

22 files changed, 62 insertions, 41 deletions

diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile
index 1f591a5cff..18c5490d7b 100644
--- a/erts/doc/src/Makefile
+++ b/erts/doc/src/Makefile
@@ -69,6 +69,7 @@ XML_PART_FILES = \
 	part.xml
 
 XML_CHAPTER_FILES = \
+	introduction.xml \
 	tty.xml \
 	match_spec.xml \
 	crash_dump.xml \
@@ -80,8 +81,7 @@ XML_CHAPTER_FILES = \
 	erl_dist_protocol.xml \
 	communication.xml \
 	time_correction.xml \
-	notes.xml \
-	notes_history.xml
+	notes.xml
 
 TOPDOCDIR=../../../doc
 
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile
index b60b04c4ae..293ef591cb 100644
--- a/lib/common_test/doc/src/Makefile
+++ b/lib/common_test/doc/src/Makefile
@@ -60,6 +60,7 @@ XML_REF6_FILES = common_test_app.xml
 XML_PART_FILES = part.xml
 
 XML_CHAPTER_FILES = \
+	introduction.xml \
 	basics_chapter.xml \
 	getting_started_chapter.xml \
 	install_chapter.xml \
@@ -74,8 +75,7 @@ XML_CHAPTER_FILES = \
 	event_handler_chapter.xml \
 	ct_hooks_chapter.xml \
 	dependencies_chapter.xml \
-	notes.xml \
-	notes_history.xml
+	notes.xml
 
 BOOK_FILES = book.xml
 
diff --git a/lib/compiler/doc/src/Makefile b/lib/compiler/doc/src/Makefile
index 254445c111..13210de040 100644
--- a/lib/compiler/doc/src/Makefile
+++ b/lib/compiler/doc/src/Makefile
@@ -39,7 +39,7 @@ XML_APPLICATION_FILES = ref_man.xml
 XML_REF3_FILES = compile.xml
 
 XML_PART_FILES =
-XML_CHAPTER_FILES = notes.xml notes_history.xml
+XML_CHAPTER_FILES = notes.xml
 
 BOOK_FILES = book.xml
 
diff --git a/lib/crypto/doc/src/Makefile b/lib/crypto/doc/src/Makefile
index 9c503b8fe0..4a0462f8e7 100644
--- a/lib/crypto/doc/src/Makefile
+++ b/lib/crypto/doc/src/Makefile
@@ -38,7 +38,7 @@ XML_APPLICATION_FILES = ref_man.xml
 XML_REF3_FILES = crypto.xml
 XML_REF6_FILES = crypto_app.xml
 
-XML_PART_FILES = release_notes.xml usersguide.xml
+XML_PART_FILES = usersguide.xml
 XML_CHAPTER_FILES = notes.xml licenses.xml fips.xml
 
 BOOK_FILES = book.xml
diff --git a/lib/debugger/doc/src/Makefile b/lib/debugger/doc/src/Makefile
index 0f724b6f17..cc0b8861d3 100644
--- a/lib/debugger/doc/src/Makefile
+++ b/lib/debugger/doc/src/Makefile
@@ -41,7 +41,7 @@ XML_APPLICATION_FILES = ref_man.xml
 XML_REF3_FILES = debugger.xml i.xml int.xml
 
 XML_PART_FILES = part.xml
-XML_CHAPTER_FILES = debugger_chapter.xml notes.xml
+XML_CHAPTER_FILES = introduction.xml debugger_chapter.xml notes.xml
 
 BOOK_FILES = book.xml
 
diff --git a/lib/eldap/doc/src/Makefile b/lib/eldap/doc/src/Makefile
index ac869e446f..aff1da4a9a 100644
--- a/lib/eldap/doc/src/Makefile
+++ b/lib/eldap/doc/src/Makefile
@@ -37,7 +37,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN)
 XML_APPLICATION_FILES = ref_man.xml
 XML_REF3_FILES = eldap.xml
 
-XML_PART_FILES = release_notes.xml usersguide.xml
+XML_PART_FILES = usersguide.xml
 XML_CHAPTER_FILES = notes.xml
 
 BOOK_FILES = book.xml
diff --git a/lib/erl_interface/doc/src/Makefile b/lib/erl_interface/doc/src/Makefile
index a96ef62786..8ef7e9648c 100644
--- a/lib/erl_interface/doc/src/Makefile
+++ b/lib/erl_interface/doc/src/Makefile
@@ -53,7 +53,7 @@ XML_APPLICATION_FILES = ref_man.xml
 #ref_man_ei.xml ref_man_erl_interface.xml
 XML_PART_FILES = \
 	part.xml
-XML_CHAPTER_FILES = ei_users_guide.xml notes.xml notes_history.xml
+XML_CHAPTER_FILES = ei_users_guide.xml notes.xml
 
 XML_FILES = $(XML_REF1_FILES) $(XML_REF3_FILES) $(BOOK_FILES) \
 	 $(XML_APPLICATION_FILES) $(XML_PART_FILES) $(XML_CHAPTER_FILES)
diff --git a/lib/hipe/doc/src/Makefile b/lib/hipe/doc/src/Makefile
index 63154abd6a..1c774d3357 100644
--- a/lib/hipe/doc/src/Makefile
+++ b/lib/hipe/doc/src/Makefile
@@ -38,7 +38,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN)
 XML_APPLICATION_FILES =  ref_man.xml
 XML_REF3_FILES =
 
-XML_PART_FILES =
+XML_PART_FILES = hipe_app.xml
 XML_CHAPTER_FILES = notes.xml
 
 BOOK_FILES = book.xml
diff --git a/lib/inets/doc/src/Makefile b/lib/inets/doc/src/Makefile
index 14f12ee949..cbfa5c9e30 100644
--- a/lib/inets/doc/src/Makefile
+++ b/lib/inets/doc/src/Makefile
@@ -39,6 +39,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/$(APPLICATION)-$(VSN)
 XML_APPLICATION_FILES = ref_man.xml
 
 XML_CHAPTER_FILES = \
+	introduction.xml \
 	inets_services.xml \
 	http_client.xml \
 	http_server.xml \
diff --git a/lib/jinterface/doc/src/Makefile b/lib/jinterface/doc/src/Makefile
index 7eb0e20b4d..37de0a35c5 100644
--- a/lib/jinterface/doc/src/Makefile
+++ b/lib/jinterface/doc/src/Makefile
@@ -46,12 +46,11 @@ XML_PART_FILES = \
 	part.xml
 XML_CHAPTER_FILES = \
 	notes.xml \
-	notes_history.xml \
 	jinterface_users_guide.xml
 
 BOOK_FILES = book.xml
 
-XML_FILES = $(BOOK_FILES) $(XML_APPLICATION_FILES) $(XML_REF3_FILES) \
+XML_FILES = $(BOOK_FILES) $(XML_APP_FILES) $(XML_REF3_FILES) \
             $(XML_PART_FILES) $(XML_CHAPTER_FILES)
 
 GIF_FILES =
diff --git a/lib/kernel/doc/src/Makefile b/lib/kernel/doc/src/Makefile
index c9d23ac4c4..0759f362d4 100644
--- a/lib/kernel/doc/src/Makefile
+++ b/lib/kernel/doc/src/Makefile
@@ -71,7 +71,7 @@ XML_REF4_FILES = app.xml config.xml
 XML_REF6_FILES = kernel_app.xml
 
 XML_PART_FILES =
-XML_CHAPTER_FILES = notes.xml notes_history.xml
+XML_CHAPTER_FILES = notes.xml
 
 BOOK_FILES = book.xml
 
diff --git a/lib/mnesia/doc/src/Makefile b/lib/mnesia/doc/src/Makefile
index da7a9e9516..82fcf66256 100644
--- a/lib/mnesia/doc/src/Makefile
+++ b/lib/mnesia/doc/src/Makefile
@@ -48,6 +48,7 @@ XML_PART_FILES = \
 
 XML_CHAPTER_FILES = \
 	Mnesia_chap1.xml \
+	Mnesia_overview.xml \
 	Mnesia_chap2.xml \
 	Mnesia_chap3.xml \
 	Mnesia_chap4.xml \
diff --git a/lib/observer/doc/src/Makefile b/lib/observer/doc/src/Makefile
index a3b0663041..11bfee1bdb 100644
--- a/lib/observer/doc/src/Makefile
+++ b/lib/observer/doc/src/Makefile
@@ -48,12 +48,12 @@ XML_PART_FILES = \
 	part.xml
 
 XML_CHAPTER_FILES = \
+	introduction_ug.xml \
 	crashdump_ug.xml \
 	etop_ug.xml \
 	observer_ug.xml \
 	ttb_ug.xml \
-	notes.xml \
-	notes_history.xml
+	notes.xml
 
 BOOK_FILES = book.xml
 
diff --git a/lib/runtime_tools/doc/src/Makefile b/lib/runtime_tools/doc/src/Makefile
index dad229e193..ec19a4ce59 100644
--- a/lib/runtime_tools/doc/src/Makefile
+++ b/lib/runtime_tools/doc/src/Makefile
@@ -45,7 +45,7 @@ XML_REF3_FILES = dbg.xml dyntrace.xml erts_alloc_config.xml system_information.x
 XML_REF6_FILES = runtime_tools_app.xml
 
 XML_PART_FILES = part.xml
-XML_CHAPTER_FILES = notes.xml notes_history.xml LTTng.xml
+XML_CHAPTER_FILES = notes.xml LTTng.xml
 
 GENERATED_XML_FILES = DTRACE.xml SYSTEMTAP.xml
 
@@ -54,7 +54,8 @@ BOOK_FILES = book.xml
 XML_FILES = \
 	$(BOOK_FILES) $(XML_CHAPTER_FILES) \
 	$(XML_PART_FILES) $(XML_REF3_FILES) \
-	$(XML_REF6_FILES) $(XML_APPLICATION_FILES)
+	$(XML_REF6_FILES) $(XML_APPLICATION_FILES) \
+	$(GENERATED_XML_FILES)
 
 GIF_FILES =
 
@@ -89,7 +90,6 @@ SPECS_FLAGS = -I../../include -I../../../kernel/src
 # ----------------------------------------------------
 # Targets
 # ----------------------------------------------------
-$(XML_FILES): $(GENERATED_XML_FILES)
 
 %.xml: $(ERL_TOP)/HOWTO/%.md $(ERL_TOP)/make/emd2exml
 	$(ERL_TOP)/make/emd2exml $< $@
diff --git a/lib/sasl/doc/src/Makefile b/lib/sasl/doc/src/Makefile
index 76746e44e7..baf563ca62 100644
--- a/lib/sasl/doc/src/Makefile
+++ b/lib/sasl/doc/src/Makefile
@@ -47,8 +47,7 @@ XML_REF6_FILES = sasl_app.xml
 XML_PART_FILES = part.xml
 XML_CHAPTER_FILES = sasl_intro.xml \
 	error_logging.xml \
-	notes.xml \
-	notes_history.xml
+	notes.xml
 
 BOOK_FILES = book.xml
 
diff --git a/lib/ssl/doc/src/Makefile b/lib/ssl/doc/src/Makefile
index d54ef47461..0e1f026d46 100644
--- a/lib/ssl/doc/src/Makefile
+++ b/lib/ssl/doc/src/Makefile
@@ -43,6 +43,7 @@ XML_REF6_FILES = ssl_app.xml
 
 XML_PART_FILES = usersguide.xml
 XML_CHAPTER_FILES = \
+	ssl_introduction.xml \
 	ssl_protocol.xml \
 	using_ssl.xml \
 	pkix_certs.xml \
diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile
index 93eac8220d..e7ea38c5c3 100644
--- a/lib/stdlib/doc/src/Makefile
+++ b/lib/stdlib/doc/src/Makefile
@@ -104,7 +104,8 @@ XML_REF3_FILES = \
 XML_REF6_FILES = stdlib_app.xml
 
 XML_PART_FILES = part.xml
-XML_CHAPTER_FILES = io_protocol.xml unicode_usage.xml notes.xml notes_history.xml assert_hrl.xml
+XML_CHAPTER_FILES = introduction.xml io_protocol.xml unicode_usage.xml \
+	notes.xml assert_hrl.xml
 
 BOOK_FILES = book.xml
 
diff --git a/lib/tools/doc/src/Makefile b/lib/tools/doc/src/Makefile
index 7011f869cd..b554781382 100644
--- a/lib/tools/doc/src/Makefile
+++ b/lib/tools/doc/src/Makefile
@@ -58,8 +58,7 @@ XML_CHAPTER_FILES = \
 	lcnt_chapter.xml \
 	erlang_mode_chapter.xml \
 	xref_chapter.xml \
-	notes.xml \
-	notes_history.xml
+	notes.xml
 
 
 BOOK_FILES = book.xml
diff --git a/make/otp_release_targets.mk b/make/otp_release_targets.mk
index ff8f881493..23b4416963 100644
--- a/make/otp_release_targets.mk
+++ b/make/otp_release_targets.mk
@@ -110,21 +110,38 @@ $(HTMLDIR)/$(APPLICATION).eix: $(XML_FILES) $(SPECS_FILES)
 	        -xinclude $(TOP_SPECS_PARAM)  \
 		-path $(DOCGEN)/priv/dtd \
 	        -path $(DOCGEN)/priv/dtd_html_entities \
-	        $(DOCGEN)/priv/xsl/db_eix.xsl book.xml >  $@ 
+	        $(DOCGEN)/priv/xsl/db_eix.xsl book.xml >  $@
 
 docs: $(HTMLDIR)/$(APPLICATION).eix
 
-xmllint: $(XML_FILES)
-	@echo "Running xmllint"
-	@BookFiles=`awk -F\" '/xi:include/ {print $$2}' book.xml`; \
-	for i in $$BookFiles; do \
-		if [ $$i = "notes.xml" ]; then \
-			echo Checking $$i; \
-			xmllint --noout --valid --nodefdtd --loaddtd --path $(DOCGEN)/priv/dtd:$(DOCGEN)/priv/dtd_html_entities $$i; \
-		else\
-			awk -F\" '/xi:include/ {print "echo Checking " $$2 ;print "xmllint --noout --valid --nodefdtd --loaddtd --path $(DOCGEN)/priv/dtd:$(DOCGEN)/priv/dtd_html_entities:$(XMLLINT_SRCDIRS) " $$2}' $$i |sh; \
-		fi \
-	done
+## Here awk is used to find all xi:include files in $(BOOK_FILES)
+## Then we look into all those files check for xi:includes
+BOOK_XI_INC_FILES:=$(foreach file,$(BOOK_FILES),$(shell awk -F\" '/xi:include/ {print $$2}' $(file))) $(BOOK_FILES)
+ALL_XI_INC_FILES:=$(foreach file,$(BOOK_XI_INC_FILES),$(shell awk -F\" '/xi:include/ {if ("$(dir $(file))" != "./") printf "$(dir $(file))"; print $$2}' $(file))) $(BOOK_XI_INC_FILES)
+
+## These are the patterns of file names that xmllint cannot currently parse
+XI_INC_FILES:=%user_man.xml %usersguide.xml %refman.xml %ref_man.xml %part.xml %book.xml
+
+## These are the files that we should run the xmllint on
+LINT_XI_INC_FILES := $(filter-out $(XI_INC_FILES), $(ALL_XI_INC_FILES))
+
+EMPTY :=
+SPACE := $(EMPTY) $(EMPTY)
+XMLLINT_SRCDIRS:=$(subst $(SPACE),:,$(sort $(foreach file,$(XML_FILES),$(dir $(file)))))
+
+xmllint: $(ALL_XI_INC_FILES)
+## We verify that the $(XML_FILES) variable in the Makefile have exactly
+## the same files as we found out by following xi:include.
+ifneq ($(filter-out $(filter %.xml,$(XML_FILES)),$(ALL_XI_INC_FILES)),)
+	$(error "$(filter-out $(filter %.xml,$(XML_FILES)),$(ALL_XI_INC_FILES)) in $$ALL_XI_INC_FILES but not in $$XML_FILES");
+endif
+ifneq ($(filter-out $(ALL_XI_INC_FILES),$(filter %.xml,$(XML_FILES))),)
+	$(error "$(filter-out $(ALL_XI_INC_FILES),$(filter %.xml,$(XML_FILES))) in $$XML_FILES but not in $$ALL_XI_INC_FILES");
+endif
+	@echo "xmllint $(LINT_XI_INC_FILES)"
+	@xmllint --noout --valid --nodefdtd --loaddtd --path \
+	$(DOCGEN)/priv/dtd:$(DOCGEN)/priv/dtd_html_entities:$(XMLLINT_SRCDIRS) \
+	$(LINT_XI_INC_FILES)
 
 # ----------------------------------------------------
 # Local documentation target for testing 
diff --git a/system/doc/efficiency_guide/xmlfiles.mk b/system/doc/efficiency_guide/xmlfiles.mk
index 88df9417f5..23c0d991b4 100644
--- a/system/doc/efficiency_guide/xmlfiles.mk
+++ b/system/doc/efficiency_guide/xmlfiles.mk
@@ -29,5 +29,5 @@ EFF_GUIDE_CHAPTER_FILES = \
 	processes.xml \
 	profiling.xml \
 	tablesDatabases.xml \
-	drivers.xml
-
+	drivers.xml \
+	retired_myths.xml
diff --git a/system/doc/reference_manual/xmlfiles.mk b/system/doc/reference_manual/xmlfiles.mk
index 61637ae701..fffcbdd911 100644
--- a/system/doc/reference_manual/xmlfiles.mk
+++ b/system/doc/reference_manual/xmlfiles.mk
@@ -30,5 +30,6 @@ REF_MAN_CHAPTER_FILES = \
 	processes.xml \
 	distributed.xml \
 	code_loading.xml \
-	ports.xml
-
+	ports.xml \
+	character_set.xml \
+	typespec.xml
diff --git a/system/doc/top/Makefile b/system/doc/top/Makefile
index 116ec688fa..b6a80aadf5 100644
--- a/system/doc/top/Makefile
+++ b/system/doc/top/Makefile
@@ -50,6 +50,8 @@ include ../tutorial/xmlfiles.mk
 include ../design_principles/xmlfiles.mk
 include ../oam/xmlfiles.mk
 
+BOOK_FILES = book.xml
+
 XML_FILES = \
 	$(INST_GUIDE_CHAPTER_FILES:%=../installation_guide/%) \
 	$(SYSTEM_PRINCIPLES_CHAPTER_FILES:%=../system_principles/%) \
@@ -70,9 +72,9 @@ XML_FILES = \
 	../efficiency_guide/part.xml \
 	../tutorial/part.xml \
 	../design_principles/part.xml \
-	../oam/part.xml
+	../oam/part.xml \
+	$(BOOK_FILES)
 
-BOOK_FILES = book.xml
 
 XMLLINT_SRCDIRS= ../installation_guide:../system_principles:../embedded:../getting_started:../reference_manual:../programming_examples:../efficiency_guide:../tutorial:../design_principles:../oam
 HTMLDIR= ../html