Merge pull request #36 from jltallon/include_dir

Add @include_dir directive: debian-style ("conf.d") config processing

AUTHORS

@@ -4,3 +4,4 @@ Daniel Marjamäki - Enhancements & bugfixes.
 Andrew Tytula - Windows port.
 Glenn Herteg - Enhancements, bugfixes, documentation corrections.
 Matt Renaud - Enhancements & bugfixes.
+JoseLuis Tallon - Enhancements & bugfixes

doc/libconfig.texi

@@ -36,7 +36,7 @@
 
 @page
 @vskip 0pt plus 1filll
-Copyright @copyright{} 2005-2014  Mark A Lindner
+Copyright @copyright{} 2005-2015  Mark A Lindner
 
 Permission is granted to make and distribute verbatim copies of
 this manual provided the copyright notice and this permission notice
@@ -599,6 +599,7 @@ not treated as part of the configuration. Therefore if the
 configuration is written back out to a stream, any comments that were
 present in the original configuration will be lost.
 
+
 @node Include Directives, , Comments, Configuration Files
 @comment  node-name,  next,  previous,  up
 @section Include Directives
@@ -640,9 +641,61 @@ info: @{
 @end smallexample
 @end cartouche
 
+The resulting configuration will be equivalent to one in which the
+contents of the ``quote.cfg`` file appeared at the point where the
+@@include directive is placed.
+
+@cindex include_dir directive
+Additionally, a directory containing configuration snippets can be
+processed using an @b{include_dir} directive. This directive has the
+effect of inlining the contents of the files contained in the named
+directory at the point of inclusion.
+The configuration files are ordered lexicographically before processing
+and ``hidden`` files (those having a name beginning with a dot) are
+skipped.
+
+An include_dir directive must appear on its own line in the input:
+
+@b{@@include_dir "}@i{dirname}@b{"}
+
+The same quoting rules as for @i{@@include} apply.
+
+@cartouche
+@smallexample
+# directory: myapp/conf.d
+00prolog.cfg
+01header.cfg
+02body.cfg
+03footer.cfg
+99epilog.cfg
+@end smallexample
+@end cartouche
+
+@cartouche
+@smallexample
+# file: test.cfg
+docinfo: @{
+  name = "Some complex config";
+  @@include_dir "myapp/conf.d"
+@};
+@end smallexample
+@end cartouche
+
+Please keep in mind that the included files will be read and processed in
+lexicographical order, as it has become customary.
+
 Include files may be nested to a maximum of 10 levels; exceeding this
 limit results in a parse error.
 
+When the path argument to an @b{@@include} or @b{@@include_dir} directive
+begins with a PATH_SEPARATOR ('/' or '\'), it is considered to be absolute.
+Alternatively, when an @i{include_basedir} has been set by means of a
+@b{config_set_include_dir} call, the directory and/or filename will be
+taken as relative to the specified include_basedir.
+Otherwise, the name is interpreted as relative to the program's CWD
+(current working directory).
+
+
 Like comments, include directives are not part of the configuration
 file syntax. They are processed before the configuration itself is
 parsed. Therefore, they are not preserved when the configuration is

lib/scanctx.c

 /* ----------------------------------------------------------------------------
    libconfig - A library for processing structured configuration files
-   Copyright (C) 2005-2014  Mark A Lindner
+   Copyright (C) 2005-2015  Mark A Lindner
 
    This file is part of libconfig.
 
@@ -26,6 +26,7 @@
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
+#include <dirent.h>
 
 #define STRING_BLOCK_SIZE 64
 #define CHUNK_SIZE 32
@@ -92,43 +93,26 @@ const char **scanctx_cleanup(struct scan_context *ctx,
 /* ------------------------------------------------------------------------- */
 
 FILE *scanctx_push_include(struct scan_context *ctx, void *buffer,
-                           const char **error)
+                            const char *file, const char **error)
 {
   FILE *fp = NULL;
-  const char *file;
-  char *full_file = NULL;
-
-  *error = NULL;
 
   if(ctx->depth == MAX_INCLUDE_DEPTH)
   {
     *error = err_include_too_deep;
     return(NULL);
   }
-
-  file = scanctx_take_string(ctx);
-  if(ctx->config->include_dir)
-  {
-    full_file = (char *)malloc(strlen(ctx->config->include_dir) + strlen(file)
-                               + 2);
-    strcpy(full_file, ctx->config->include_dir);
-    strcat(full_file, FILE_SEPARATOR);
-    strcat(full_file, file);
-  }
-
-  fp = fopen(full_file ? full_file : file, "rt");
-  free((void *)full_file);
-
+  fp = fopen(file, "rt");
   if(fp)
   {
     ctx->streams[ctx->depth] = fp;
     ctx->files[ctx->depth] = __scanctx_add_filename(ctx, file);
     ctx->buffers[ctx->depth] = buffer;
     ++(ctx->depth);
+      *error = NULL;
   }
   else
   {
-    free((void *)file);
     *error = err_bad_include;
   }
 
@@ -137,6 +121,40 @@ FILE *scanctx_push_include(struct scan_context *ctx, void *buffer,
 
 /* ------------------------------------------------------------------------- */
 
+const char *scanctx_getpath(struct scan_context *ctx)
+{
+  const char *name;
+  const char *full_path = NULL;
+
+  name = scanctx_take_string(ctx);
+  if((name[0] != FILE_SEPARATOR[0]) && ctx->config->include_dir)
+  {
+    full_path = scanctx_filename(ctx, ctx->config->include_dir, name);
+    free((void*)name);
+  }
+  else
+    full_path = name;
+
+  return full_path;
+}
+
+
+/* ------------------------------------------------------------------------- */
+
+const char *scanctx_filename(struct scan_context *ctx,const char* dirname, const char* filename)
+{
+  const char *basedir = dirname ? dirname : ctx->basedir;
+  char *full_file = (char *)malloc(strlen(basedir) + strlen(filename) + 2);
+
+  strcpy(full_file, basedir);
+  strcat(full_file, FILE_SEPARATOR);
+  strcat(full_file, filename);
+
+  return full_file;
+}
+
+/* ------------------------------------------------------------------------- */
+
 void *scanctx_pop_include(struct scan_context *ctx)
 {
   void *buffer;
@@ -146,13 +164,69 @@ void *scanctx_pop_include(struct scan_context *ctx)
 
   --(ctx->depth);
   buffer = ctx->buffers[ctx->depth];
-  fclose(ctx->streams[ctx->depth]);
+  if(ctx->streams[ctx->depth])
+    fclose(ctx->streams[ctx->depth]);
 
   return(buffer);
 }
 
 /* ------------------------------------------------------------------------- */
 
+extern const char *scanctx_dirnext(struct scan_context* ctx)
+{
+  struct dirent** dentries = (struct dirent**)ctx->dentries;
+  if(!ctx->dentries || (ctx->de_cur == ctx->de_max))	/* shouldn't happen.... */
+    return NULL;
+
+  return dentries[ctx->de_cur++]->d_name;
+}
+
+int scanctx_dirscan(struct scan_context* ctx, const char* dirname,
+                    int (*filter)(const struct dirent *),
+                    int (*compar)(const struct dirent **, const struct dirent **))
+{
+  int n;
+
+  if(!dirname)
+    return -1;
+
+  ctx->dentries = NULL;
+  if((n = scandir(dirname, (struct dirent***)&ctx->dentries, filter, compar)) < 0)
+    return n;
+
+  ctx->basedir = dirname;
+  ctx->de_max = n;
+  ctx->de_cur = 0;
+
+  return n;
+}
+
+void scanctx_dirend(struct scan_context* ctx)
+{
+  struct dirent **dentries = (struct dirent**)ctx->dentries;
+  unsigned i;
+
+  for(i = 0; i < ctx->de_max; i++)
+    free(dentries[i]);
+  free(ctx->dentries);
+  ctx->dentries = NULL;
+
+  if(ctx->basedir)
+    free((void*)ctx->basedir);
+
+  ctx->de_cur = ctx->de_max = 0;
+}
+
+int scanctx_inloop(const struct scan_context* ctx)
+{
+  if(!ctx->dentries)
+    return 0;
+
+  return (ctx->de_cur < ctx->de_max) ? 1 : 0;
+}
+
+/* ------------------------------------------------------------------------- */
+
 char *scanctx_take_string(struct scan_context *ctx)
 {
   char *r = strbuf_release(&(ctx->string));

lib/scanctx.h

@@ -38,18 +38,33 @@ struct scan_context
   const char *files[MAX_INCLUDE_DEPTH];
   void *buffers[MAX_INCLUDE_DEPTH];
   FILE *streams[MAX_INCLUDE_DEPTH];
-  int depth;
   strbuf_t string;
   const char **filenames;
   unsigned int num_filenames;
+  int depth;
+  void **dentries;		/* dirent** */
+  const char *basedir;	/* basedir for @include_dir */
+  unsigned de_max, de_cur;	/* counters into dirent* array */
 };
 
+struct dirent;	/* forward decl */
+
 extern void scanctx_init(struct scan_context *ctx, const char *top_filename);
 extern const char **scanctx_cleanup(struct scan_context *ctx,
                                     unsigned int *num_filenames);
 
+extern const char *scanctx_getpath(struct scan_context *ctx);
+extern const char *scanctx_filename(struct scan_context *ctx, const char *dirname, const char *filename);
+
+extern const char* scanctx_dirnext(struct scan_context* ctx);
+extern int   scanctx_dirscan(struct scan_context* ctx, const char* dirname,
+				int (*filter)(const struct dirent *),
+				int (*compar)(const struct dirent **, const struct dirent **));
+extern void  scanctx_dirend(struct scan_context* ctx);
+extern int   scanctx_inloop(const struct scan_context* ctx);
+
 extern FILE *scanctx_push_include(struct scan_context *ctx, void *prev_buffer,
-                                  const char **error);
+                                  const char *file, const char **error);
 extern void *scanctx_pop_include(struct scan_context *ctx);
 
 #define scanctx_append_string(C, S)             \

lib/scanner.l

@@ -42,6 +42,7 @@
 #include <ctype.h>
 #include <string.h>
 #include <limits.h>
+#include <dirent.h>
 #include "parsectx.h"
 #include "scanctx.h"
 #include "grammar.h"
@@ -83,6 +84,22 @@ static unsigned long long fromhex(const char *s)
 #endif /* __MINGW32__ */
 }
 
+static int filter_dotfiles(const struct dirent *de)
+{
+  const char *fname = de->d_name;
+
+#ifdef _DIRENT_HAVE_D_TYPE
+  /* filter out non-files and non-symlinks */
+  if((de->d_type != DT_REG) && (de->d_type != DT_LNK) && (de->d_type != DT_UNKNOWN))
+    return 0;
+#endif
+
+  return (fname /* != NULL */
+    && ('\0' != fname[0]) /* can't really happen */
+    && ('.' != fname[0])
+  ) ? 1 : 0 ;
+}
+
 %}
 
 true              [Tt][Rr][Uu][Ee]
@@ -95,9 +112,10 @@ hex64             0[Xx][0-9A-Fa-f]+L(L)?
 hexchar           \\[Xx][0-9A-Fa-f]{2}
 float             ([-+]?([0-9]*)?\.[0-9]*([eE][-+]?[0-9]+)?)|([-+]?([0-9]+)(\.[0-9]*)?[eE][-+]?[0-9]+)
 comment           (#|\/\/).*$
-include_open      ^[ \t]*@include[ \t]+\"
+include_file_open ^[ \t]*@include[ \t]+\"
+include_dir_open  ^[ \t]*@include_dir[ \t]+\"
 
-%x COMMENT STRING INCLUDE
+%x COMMENT STRING INCLUDE_F INCLUDE_D
 
 %%
 
@@ -126,33 +144,80 @@ include_open      ^[ \t]*@include[ \t]+\"
                     return(TOK_STRING);
                   }
 
-{include_open}    { BEGIN INCLUDE; }
-<INCLUDE>[^\"\\]+ { scanctx_append_string(yyextra, yytext); }
-<INCLUDE>\\\\     { scanctx_append_string(yyextra, "\\"); }
-<INCLUDE>\\\"     { scanctx_append_string(yyextra, "\""); }
-<INCLUDE>\"       {
-                    const char *error;
-                    FILE *fp = scanctx_push_include(yyextra,
-                                                    (void *)YY_CURRENT_BUFFER,
-                                                    &error);
-                    if(fp)
-                    {
-                      yyin = fp;
-                      yy_switch_to_buffer(
-                        yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner),
-                        yyscanner);
-                    }
-                    else
-                    {
-                      yyextra->config->error_text = error;
-                      yyextra->config->error_file = scanctx_current_filename(
-                        yyextra);
-                      yyextra->config->error_line = libconfig_yyget_lineno(
-                        yyscanner);
-                      return TOK_ERROR;
-                    }
-                    BEGIN INITIAL;
-                  }
+{include_file_open}     { BEGIN INCLUDE_F; }
+<INCLUDE_F>[^\"\\]+     { scanctx_append_string(yyextra, yytext); }
+<INCLUDE_F>\\\\         { scanctx_append_string(yyextra, "\\"); }
+<INCLUDE_F>\\\"         { scanctx_append_string(yyextra, "\""); }
+<INCLUDE_F>\"           {
+      const char *error;
+      FILE *fp = scanctx_push_include(yyextra,
+        (void *)YY_CURRENT_BUFFER,
+        scanctx_getpath(yyextra),
+        &error);
+      if(fp)
+      {
+        yyin = fp;
+        yy_switch_to_buffer(
+          yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner),
+          yyscanner
+        );
+      }
+      else
+      {
+        yyextra->config->error_text = error;
+        yyextra->config->error_file = scanctx_current_filename(
+          yyextra);
+        yyextra->config->error_line = libconfig_yyget_lineno(yyscanner);
+        return TOK_ERROR;
+      }
+      BEGIN INITIAL;
+    }
+
+{include_dir_open}      { BEGIN INCLUDE_D; }
+<INCLUDE_D>[^\"\\]+     { scanctx_append_string(yyextra, yytext); }
+<INCLUDE_D>\\\\         { scanctx_append_string(yyextra, "\\"); }
+<INCLUDE_D>\\\"         { scanctx_append_string(yyextra, "\""); }
+<INCLUDE_D>\"           {
+        const char *error;
+        const char* basedir;
+        FILE *fp = NULL;
+
+        basedir = scanctx_getpath(yyextra);
+        if(scanctx_dirscan(yyextra, basedir, filter_dotfiles, alphasort) < 0)
+        {
+          if(basedir)
+            free((void*)basedir);
+          return TOK_ERROR;
+        }
+
+        if(scanctx_inloop(yyextra))
+        {
+          fp = scanctx_push_include(yyextra,
+            (void *)YY_CURRENT_BUFFER,
+            scanctx_filename(yyextra, NULL, scanctx_dirnext(yyextra)),
+            &error
+          );
+
+          if(fp)
+          {
+            yyin = fp;
+            yy_switch_to_buffer(
+              yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner),
+              yyscanner
+            );
+          }
+          else
+          {
+            yyextra->config->error_text = error;
+            yyextra->config->error_file = scanctx_current_filename(yyextra);
+            yyextra->config->error_line = libconfig_yyget_lineno(yyscanner);
+          }
+        }
+        else
+          scanctx_dirend(yyextra);	/* avoid leaks */
+
+        BEGIN INITIAL;
+    }
 
 \n|\r|\f          { /* ignore */ }
 [ \t]+            { /* ignore */ }
@@ -170,13 +235,13 @@ include_open      ^[ \t]*@include[ \t]+\"
                     llval = atoll(yytext);
                     if((llval < INT_MIN) || (llval > INT_MAX))
                     {
-	              yylval->llval = llval;
-	              return(TOK_INTEGER64);
+                      yylval->llval = llval;
+                      return(TOK_INTEGER64);
                     }
                     else
                     {
-	              yylval->ival = llval;
-	              return(TOK_INTEGER);
+                      yylval->ival = llval;
+                      return(TOK_INTEGER);
                     }
                   }
 {integer64}       { yylval->llval = atoll(yytext); return(TOK_INTEGER64); }
@@ -194,13 +259,40 @@ include_open      ^[ \t]*@include[ \t]+\"
 .                 { return(TOK_GARBAGE); }
 
 <<EOF>>           {
-                    YY_BUFFER_STATE buf = (YY_BUFFER_STATE)scanctx_pop_include(
-                      yyextra);
-                    if(buf)
-                    {
-                      yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner);
-                      yy_switch_to_buffer(buf, yyscanner);
-                    }
-                    else
-                      yyterminate();
-                  }
+        const char* error;
+        FILE *fp;
+        YY_BUFFER_STATE buf = (YY_BUFFER_STATE)scanctx_pop_include(yyextra);
+        if(buf)
+        {
+          yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner);
+          yy_switch_to_buffer(buf, yyscanner);
+        }
+        else	/* if no more buffers, we are done */
+          yyterminate();
+
+        if(scanctx_inloop(yyextra))
+        {
+          /* gotta keep looping.... */
+          fp = scanctx_push_include(yyextra,
+            (void *)YY_CURRENT_BUFFER,
+            scanctx_filename(yyextra, NULL, scanctx_dirnext(yyextra)),
+            &error
+          );
+          if(fp)
+          {
+            yyin = fp;
+            yy_switch_to_buffer(
+              yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner),
+              yyscanner
+            );
+          }
+          else
+          {
+            yyextra->config->error_text = error;
+            yyextra->config->error_file = scanctx_current_filename(yyextra);
+            yyextra->config->error_line = libconfig_yyget_lineno(yyscanner);
+          }
+        }
+        else	/* not on loop, or just finished */
+          scanctx_dirend(yyextra);
+    }