erl_docgen: Add new internal docs chapter to docs

10 files changed, 173 insertions, 26 deletions

diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile
index 06a8691c0e..8f0e47f85f 100644
--- a/erts/doc/src/Makefile
+++ b/erts/doc/src/Makefile
@@ -67,7 +67,21 @@ XML_REF3_FILES = \
 	erts_alloc.xml
 
 XML_PART_FILES = \
-	part.xml
+	part.xml internal.xml
+
+XML_INTERNAL_FILES = \
+	CarrierMigration.xml	\
+	ThreadProgress.xml	\
+	CodeLoading.xml		\
+	Tracing.xml		\
+	DelayedDealloc.xml	\
+	beam_makeops.xml		\
+	GarbageCollection.xml	\
+	PTables.xml		\
+	PortSignals.xml		\
+	ProcessManagementOptimizations.xml \
+	SuperCarrier.xml
+
 
 XML_CHAPTER_FILES = \
 	introduction.xml \
@@ -97,6 +111,8 @@ XML_FILES = \
 	$(BOOK_FILES) $(XML_CHAPTER_FILES) \
 	$(XML_PART_FILES) $(XML_REF3_FILES) $(XML_REF1_FILES) $(XML_APPLICATION_FILES)
 
+XML_GEN_FILES = $(XML_INTERNAL_FILES:%=$(XMLDIR)/%)
+
 # ----------------------------------------------------
 
 HTML_FILES = $(XML_APPLICATION_FILES:%.xml=$(HTMLDIR)/%.html) \
@@ -164,6 +180,9 @@ $(SPECDIR)/specs_%.xml:
 	$(gen_verbose)escript $(SPECS_EXTRACTOR) $(SPECS_FLAGS) \
                 -o$(dir $@) -module $(patsubst $(SPECDIR)/specs_%.xml,%,$@)
 
+$(XMLDIR)/%.xml: ../../emulator/internal_doc/%.md $(ERL_TOP)/make/emd2exml
+	$(ERL_TOP)/make/emd2exml $< $@
+
 # ----------------------------------------------------
 # Release Target
 # ----------------------------------------------------
diff --git a/erts/doc/src/book.xml b/erts/doc/src/book.xml
index a0780c91d9..d79da1e4f7 100644
--- a/erts/doc/src/book.xml
+++ b/erts/doc/src/book.xml
@@ -41,6 +41,9 @@
   <applications>
     <xi:include href="ref_man.xml"/>
   </applications>
+  <internals>
+    <xi:include href="internal.xml"/>
+  </internals>
   <releasenotes>
     <xi:include href="notes.xml"/>
   </releasenotes>
diff --git a/erts/doc/src/internal.xml b/erts/doc/src/internal.xml
new file mode 100644
index 0000000000..e5f3d9f3fd
--- /dev/null
+++ b/erts/doc/src/internal.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE part SYSTEM "part.dtd">
+
+<internal xmlns:xi="http://www.w3.org/2001/XInclude">
+  <header>
+    <copyright>
+      <year>2018</year><year>2018</year>
+      <holder>Ericsson AB. All Rights Reserved.</holder>
+    </copyright>
+    <legalnotice>
+      Licensed under the Apache License, Version 2.0 (the "License");
+      you may not use this file except in compliance with the License.
+      You may obtain a copy of the License at
+ 
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+      See the License for the specific language governing permissions and
+      limitations under the License.
+    
+    </legalnotice>
+
+    <title>ERTS Internal Documentation</title>
+    <prepared>Lukas Larsson</prepared>
+    <docno></docno>
+    <date>2018-07-07</date>
+    <rev>4.5.2</rev>
+    <file>internal.xml</file>
+  </header>
+  <description>
+  </description>
+  <xi:include href="CarrierMigration.xml"/>
+  <xi:include href="ThreadProgress.xml"/>
+  <xi:include href="CodeLoading.xml"/>
+  <xi:include href="Tracing.xml"/>
+  <xi:include href="DelayedDealloc.xml"/>
+<!--  <xi:include href="beam_makeops.xml"/>
+  <xi:include href="GarbageCollection.xml"/> -->
+  <xi:include href="PTables.xml"/>
+  <xi:include href="PortSignals.xml"/>
+  <xi:include href="ProcessManagementOptimizations.xml"/>
+  <xi:include href="SuperCarrier.xml"/> -->
+</internal>
+
diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md
index bb3d8aac28..40f6031ca8 100644
--- a/erts/emulator/internal_doc/CarrierMigration.md
+++ b/erts/emulator/internal_doc/CarrierMigration.md
@@ -1,6 +1,9 @@
 Carrier Migration
 =================
 
+Introduction
+------------
+
 The ERTS memory allocators manage memory blocks in two types of raw
 memory chunks. We call these chunks of raw memory
 *carriers*. Single-block carriers which only contain one large block,
@@ -141,11 +144,11 @@ Since the carrier has been unlinked from the data structure of
 available free blocks, no more allocations will be made in the
 carrier.
 
-The allocator instance that created a carrier is called its **owner**.
+The allocator instance that created a carrier is called its *owner*.
 Ownership never changes.
 
 The allocator instance that has the responsibility to perform deallocations in a
-carrier is called its **employer**. The employer may also perform allocations if
+carrier is called its *employer*. The employer may also perform allocations if
 the carrier is not in the pool. Employment may change when a carrier is fetched from
 or inserted into the pool.
 
@@ -153,14 +156,14 @@ Deallocations in a carrier, while it remains in the pool, is always performed
 the owner. That is, all pooled carriers are employed by their owners.
 
 Each carrier has an atomic word containing a pointer to the employing allocator
-instance and three bit flags; IN_POOL, BUSY and HOMECOMING.
+instance and three bit flags; IN\_POOL, BUSY and HOMECOMING.
 
 When fetching a carrier from the pool, employment may change and further
 deallocations in the carrier will be redirected to the new
 employer using the delayed dealloc functionality.
 
 When a foreign allocator instance abandons a carrier back into the pool, it will
-also pass it back to its **owner** using the delayed dealloc queue. When doing
+also pass it back to its *owner* using the delayed dealloc queue. When doing
 this it will set the HOMECOMING bit flag to mark it as "enqueued". The owner
 will later clear the HOMECOMING bit when the carrier is dequeued. This mechanism
 prevents a carrier from being enqueued again before it has been dequeued.
@@ -180,14 +183,14 @@ back to the owner for deallocation using the delayed dealloc functionality.
 
 In short:
 
-* The allocator instance that created a carrier **owns** it.
-* An empty carrier is always deallocated by its **owner**.
-* **Ownership** never changes.
-* The allocator instance that uses a carrier **employs** it.
-* An **employer** can abandon a carrier into the pool.
+* The allocator instance that created a carrier *owns* it.
+* An empty carrier is always deallocated by its *owner*.
+* *Ownership* never changes.
+* The allocator instance that uses a carrier *employs* it.
+* An *employer* can abandon a carrier into the pool.
 * Pooled carriers are not allocated from.
-* Pooled carriers are always **employed** by their **owner**.
-* **Employment** can only change from **owner** to a foreign allocator
+* Pooled carriers are always *employed* by their *owner*.
+* *Employment* can only change from *owner* to a foreign allocator
   when a carrier is fetched from the pool.
 
 
@@ -229,7 +232,7 @@ carrier. When the cluster gets to the same size as the search limit,
 all searches will essentially fail.
 
 To counter the "bad cluster" problem and also ease the contention, the
-search will now always start by first looking at the allocators **own**
+search will now always start by first looking at the allocators *own*
 carriers. That is, carriers that were initially created by the
 allocator itself and later had been abandoned to the pool. If none of
 our own abandoned carrier would do, then the search continues into the
diff --git a/erts/emulator/internal_doc/CodeLoading.md b/erts/emulator/internal_doc/CodeLoading.md
index 151b9cd57c..0b2e3070e7 100644
--- a/erts/emulator/internal_doc/CodeLoading.md
+++ b/erts/emulator/internal_doc/CodeLoading.md
@@ -45,7 +45,7 @@ free to schedule other work while the second loader is waiting. (See
 `erts_release_code_write_permission`).
 
 The ability to prepare several modules in parallel is not currently
-used as almost all code loading is serialized by the code_server
+used as almost all code loading is serialized by the code\_server
 process. The BIF interface is however prepared for this.
 
       erlang:prepare_loading(Module, Code) -> LoaderState
@@ -71,8 +71,8 @@ structures. These *code access structures* are
 
 * Export table. One entry for every exported function.
 * Module table. One entry for each loaded module.
-* "beam_catches". Identifies jump destinations for catch instructions.
-* "beam_ranges". Map code address to function and line in source file.
+* "beam\_catches". Identifies jump destinations for catch instructions.
+* "beam\_ranges". Map code address to function and line in source file.
 
 The most frequently used of these structures is the export table that
 is accessed in run time for every executed external function call to
diff --git a/erts/emulator/internal_doc/PTables.md b/erts/emulator/internal_doc/PTables.md
index 6fe0e7665d..ef61963a40 100644
--- a/erts/emulator/internal_doc/PTables.md
+++ b/erts/emulator/internal_doc/PTables.md
@@ -85,13 +85,13 @@ following:
 3.  Depending on use, issue appropriate memory barrier.
 
     A common barrier used is a barrier with acquire semantics. On
-    x86/x86_64 this maps to a compiler barrier preventing the compiler
+    x86/x86\_64 this maps to a compiler barrier preventing the compiler
     to reorder instructions, but on other hardware often some kind of
     light weight hardware memory barrier is also needed.
 
     When comparing with a locked approach, at least one heavy weight
     memory barrier will be issued when locking the lock on most, if
-    not all, hardware architectures (including x86/x86_64), and often
+    not all, hardware architectures (including x86/x86\_64), and often
     some kind of light weight memory barrier will be issued when
     unlocking the lock. 
 
diff --git a/erts/emulator/internal_doc/SuperCarrier.md b/erts/emulator/internal_doc/SuperCarrier.md
index acf722ea37..f52c6613d5 100644
--- a/erts/emulator/internal_doc/SuperCarrier.md
+++ b/erts/emulator/internal_doc/SuperCarrier.md
@@ -5,7 +5,7 @@ A super carrier is large memory area, allocated at VM start, which can
 be used during runtime to allocate normal carriers from.
 
 The super carrier feature was introduced in OTP R16B03. It is
-enabled with command line option +MMscs <size in Mb>
+enabled with command line option +MMscs &lt;size in Mb&gt;
 and can be configured with other options.
 
 Problem
@@ -65,7 +65,7 @@ carrier is full.
 
 ### Implementation ###
 
-The entire super carrier implementation is kept in erl_mmap.c. The
+The entire super carrier implementation is kept in erl\_mmap.c. The
 name suggest that it can be viewed as our own mmap implementation.
 
 A super carrier needs to satisfy two slightly different kinds of
@@ -98,8 +98,8 @@ other.
 
 ### Data structures ###
 
-The MBC area is called **sa** as in super aligned and the SBC area is
-called **sua** as in super un-aligned.
+The MBC area is called *sa* as in super aligned and the SBC area is
+called *sua* as in super un-aligned.
 
 Note that the "super" in super alignment and the "super" in super
 carrier has nothing to do with each other. We could have choosen
@@ -128,7 +128,7 @@ down or up.
 We need to keep track of all the free segments in order to reuse them
 for new carrier allocations. One initial idea was to use the same
 mechanism that is used to keep track of free blocks within MBCs
-(alloc_util and the different strategies). However, that would not be
+(alloc\_util and the different strategies). However, that would not be
 as straight forward as one can think and can also waste quite a lot of
 memory as it uses prepended block headers. The granularity of the
 super carrier is one memory page (usually 4kb). We want to allocate
diff --git a/erts/emulator/internal_doc/Tracing.md b/erts/emulator/internal_doc/Tracing.md
index 7f97f64765..1b3b0fe1b8 100644
--- a/erts/emulator/internal_doc/Tracing.md
+++ b/erts/emulator/internal_doc/Tracing.md
@@ -82,7 +82,7 @@ through when adding a new breakpoint.
    instruction word in the breakpoint.
 
 3. Write a pointer to the breakpoint at offset -4 from the first
-   instruction "func_info" header.
+   instruction "func\_info" header.
 
 4. Set the staging part of the breakpoint as enabled with specified
    breakpoint data.
@@ -139,7 +139,7 @@ and removing breakpoints.
 
 2. Allocate new breakpoint structures with a disabled active part and
    the original beam instruction. Write a pointer to the breakpoint in
-   "func_info" header at offset -4.
+   "func\_info" header at offset -4.
 
 3. Update the staging part of all affected breakpoints. Disable
    breakpoints that are to be removed.
diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl
index c9be926e1e..4351c324ca 100644
--- a/lib/erl_docgen/priv/xsl/db_html.xsl
+++ b/lib/erl_docgen/priv/xsl/db_html.xsl
@@ -836,6 +836,10 @@
       <!-- .../part -->
       <xsl:call-template name="part.content" />
     </xsl:if>
+    <xsl:if test="$lname = 'internal'">
+      <!-- .../internals -->
+      <xsl:call-template name="internal.content" />
+    </xsl:if>
     <xsl:if test="$lname = 'chapter'">
       <!-- .../part/chapter -->
       <xsl:call-template name="chapter.content">
@@ -859,11 +863,17 @@
     <xsl:param name="chapnum"/>
     <xsl:param name="curModule"/>
     <xsl:if test="(local-name() = 'part') or ((local-name() = 'chapter') and ancestor::part)">
-      <!-- .../part or.../part/chapter  -->
+      <!-- .../part or .../part/chapter  -->
       <xsl:call-template name="menu.ug">
         <xsl:with-param name="chapnum" select="$chapnum"/>
       </xsl:call-template>
     </xsl:if>
+    <xsl:if test="(local-name() = 'internal') or ((local-name() = 'chapter') and ancestor::internal)">
+      <!-- .../internal or .../internal/chapter  -->
+      <xsl:call-template name="menu.internal">
+        <xsl:with-param name="chapnum" select="$chapnum"/>
+      </xsl:call-template>
+    </xsl:if>
     <xsl:if test="(local-name() = 'application') or (local-name() = 'erlref')or (local-name() = 'comref')or (local-name() = 'cref')or (local-name() = 'fileref')or (local-name() = 'appref')">
       <!-- .../application,.../application/erlref, .../application/comref or .../application/cref  or .../application/fileref or .../application/appref -->
       <xsl:call-template name="menu.ref">
@@ -902,6 +912,9 @@
       <xsl:if test="boolean(/book/applications)">
           <li><a href="index.html">Reference Manual</a></li>
       </xsl:if>
+      <xsl:if test="boolean(/book/internals)">
+          <li><a href="internal_docs.html">Internal Documentation</a></li>
+      </xsl:if>
       <xsl:if test="boolean(/book/releasenotes)">
           <li><a href="release_notes.html">Release Notes</a></li>
       </xsl:if>
@@ -942,6 +955,7 @@
   <xsl:template match="/book">
     <xsl:apply-templates select="parts"/>
     <xsl:apply-templates select="applications"/>
+    <xsl:apply-templates select="internals"/>
     <xsl:apply-templates select="releasenotes"/>
   </xsl:template>
 
@@ -955,6 +969,11 @@
     <xsl:apply-templates select="application"/>
   </xsl:template>
 
+  <!-- Internals -->
+  <xsl:template match="internals">
+    <xsl:apply-templates select="internal"/>
+  </xsl:template>
+
  <!-- Header -->
  <xsl:template match="header"/>
 
@@ -1311,6 +1330,61 @@
   </xsl:template>
 
 
+  <!-- Internal Docs -->
+
+  <!-- Part -->
+  <xsl:template match="internal">
+
+    <xsl:document href="{$outdir}/internal_docs.html" method="html"  encoding="UTF-8" indent="yes" doctype-public="-//W3C//DTD HTML 4.01 Transitional//EN">
+      <xsl:call-template name="pagelayout"/>
+    </xsl:document>
+  </xsl:template>
+
+
+  <!-- Part content-->
+  <xsl:template name="internal.content">
+    <div class="frontpage"/>
+
+    <center><h1><xsl:value-of select="/book/header/title"/> Internal Docs</h1></center>
+
+    <center><h4>Version <xsl:value-of select="$appver"/></h4></center>
+    <center><h4><xsl:value-of select="$gendate"/></h4></center>
+    <div class="extrafrontpageinfo">
+    <center><xsl:value-of select="$extra_front_page_info"/></center>
+    </div>
+
+    <xsl:apply-templates select="chapter"/>
+
+  </xsl:template>
+
+  <!-- Menu.ug -->
+  <xsl:template name="menu.internal">
+    <xsl:param name="chapnum"/>
+
+    <div id="leftnav">
+      <div class="innertube">
+
+        <xsl:call-template name="erlang_logo"/>
+
+        <p class="section-title"><xsl:value-of select="/book/header/title"/></p>
+        <p class="section-subtitle">Internal Documentation</p>
+        <p class="section-version">Version <xsl:value-of select="$appver"/></p>
+
+        <xsl:call-template name="menu_top"/>
+
+        <xsl:call-template name="menu_middle"/>
+
+        <h3>Chapters</h3>
+
+        <ul class="flipMenu" imagepath="{$topdocdir}/js/flipmenu">
+          <xsl:call-template name="menu.chapter">
+            <xsl:with-param name="entries" select="/book/internals/internal/chapter[header/title]"/>
+            <xsl:with-param name="chapnum" select="$chapnum"/>
+          </xsl:call-template>
+        </ul>
+      </div>
+    </div>
+  </xsl:template>
 
 
   <!--Users Guide -->
diff --git a/system/doc/installation_guide/Makefile b/system/doc/installation_guide/Makefile
index 38252757d6..c5234c1c9a 100644
--- a/system/doc/installation_guide/Makefile
+++ b/system/doc/installation_guide/Makefile
@@ -113,6 +113,8 @@ clean clean_docs:
 	rm -f $(GENERATED_XML_FILES)
 	rm -f $(XMLDIR)/*.xml
 	rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html
+	rm -f $(XML_GEN_FILES)
+	rm -rf $(HTMLDIR)
 	rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo)
 	rm -f errs core *~