Add @include_dir directive: debian-style ("conf.d") configuration processing

  Quite some refactoring of the includefile stack and filename processing logic

  Adds some extra fields to scan_context, but does not change the API
  "symbols" file should hide all functions called scanctx_* anyway

lib/scanctx.c

@@ -26,6 +26,7 @@
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
+#include <dirent.h>
 
 #define STRING_BLOCK_SIZE 64
 #define CHUNK_SIZE 32
@@ -92,43 +93,26 @@ const char **scanctx_cleanup(struct scan_context *ctx,
 /* ------------------------------------------------------------------------- */
 
 FILE *scanctx_push_include(struct scan_context *ctx, void *buffer,
-                           const char **error)
+				const char *file, const char **error)
 {
   FILE *fp = NULL;
-  const char *file;
-  char *full_file = NULL;
-
-  *error = NULL;
 
   if(ctx->depth == MAX_INCLUDE_DEPTH)
   {
     *error = err_include_too_deep;
     return(NULL);
   }
-
-  file = scanctx_take_string(ctx);
-  if(ctx->config->include_dir)
-  {
-    full_file = (char *)malloc(strlen(ctx->config->include_dir) + strlen(file)
-                               + 2);
-    strcpy(full_file, ctx->config->include_dir);
-    strcat(full_file, FILE_SEPARATOR);
-    strcat(full_file, file);
-  }
-
-  fp = fopen(full_file ? full_file : file, "rt");
-  free((void *)full_file);
-
+  fp = fopen(file, "rt");
   if(fp)
   {
     ctx->streams[ctx->depth] = fp;
     ctx->files[ctx->depth] = __scanctx_add_filename(ctx, file);
     ctx->buffers[ctx->depth] = buffer;
     ++(ctx->depth);
+	*error = NULL;
   }
   else
   {
-    free((void *)file);
     *error = err_bad_include;
   }
 
@@ -137,6 +121,37 @@ FILE *scanctx_push_include(struct scan_context *ctx, void *buffer,
 
 /* ------------------------------------------------------------------------- */
 
+const char *scanctx_getpath(struct scan_context *ctx)
+{
+  const char *name;
+  const char *full_path = NULL;
+
+	name = scanctx_take_string(ctx);
+	if(ctx->config->include_dir)
+		full_path = scanctx_filename(ctx, ctx->config->include_dir, name);
+	else
+		full_path = strdup(name);
+	free((void*)name);
+	return full_path;
+}
+
+
+/* ------------------------------------------------------------------------- */
+
+const char *scanctx_filename(struct scan_context *ctx,const char* dirname, const char* filename)
+{
+  const char* basedir = (NULL!=dirname)? dirname : ctx->basedir;
+  char *full_file = (char *)malloc(strlen(basedir) + strlen(filename) + 2);
+
+	strcpy(full_file, basedir);
+	strcat(full_file, FILE_SEPARATOR);
+	strcat(full_file, filename);
+
+	return full_file;
+}
+
+/* ------------------------------------------------------------------------- */
+
 void *scanctx_pop_include(struct scan_context *ctx)
 {
   void *buffer;
@@ -146,13 +161,69 @@ void *scanctx_pop_include(struct scan_context *ctx)
 
   --(ctx->depth);
   buffer = ctx->buffers[ctx->depth];
-  fclose(ctx->streams[ctx->depth]);
+  if(NULL!=ctx->streams[ctx->depth])
+	  fclose(ctx->streams[ctx->depth]);
 
   return(buffer);
 }
 
 /* ------------------------------------------------------------------------- */
 
+extern const char* scanctx_dirnext(struct scan_context* ctx)
+{
+	struct dirent** dentries= (struct dirent**)ctx->dentries;
+	if( NULL == ctx->dentries || ctx->de_cur==ctx->de_max )	/* shouldn't happen.... */
+		return NULL;
+
+	return dentries[ ctx->de_cur++ ]->d_name;
+}
+
+int scanctx_dirscan(struct scan_context* ctx, const char* dirname,
+				int (*filter)(const struct dirent *),
+				int (*compar)(const struct dirent **, const struct dirent **))
+{
+	int n;
+
+	if( NULL == dirname )
+		return -1;
+
+	ctx->dentries=NULL;
+	if( (n=scandir(dirname,(struct dirent***)&ctx->dentries,filter,compar)) < 0)
+		return n;
+
+	ctx->basedir=dirname;
+	ctx->de_max=n;
+	ctx->de_cur=0;
+
+	return n;
+}
+
+int scanctx_dirend(struct scan_context* ctx)
+{
+ struct dirent** dentries= (struct dirent**)ctx->dentries;
+ unsigned i;
+
+	for(i=0; i<ctx->de_max; i++)
+		free(dentries[i]);
+	free(ctx->dentries);
+	ctx->dentries=NULL;
+
+	if(ctx->basedir)
+		free((void*)ctx->basedir);
+	ctx->de_cur=ctx->de_max=0;
+
+	return 0;
+}
+int scanctx_inloop(const struct scan_context* ctx)
+{
+	if( NULL == ctx->dentries )
+		return 0;
+
+	return (ctx->de_cur < ctx->de_max)? 1 : 0;
+}
+
+/* ------------------------------------------------------------------------- */
+
 char *scanctx_take_string(struct scan_context *ctx)
 {
   char *r = strbuf_release(&(ctx->string));

lib/scanctx.h

@@ -38,18 +38,33 @@ struct scan_context
   const char *files[MAX_INCLUDE_DEPTH];
   void *buffers[MAX_INCLUDE_DEPTH];
   FILE *streams[MAX_INCLUDE_DEPTH];
-  int depth;
   strbuf_t string;
   const char **filenames;
   unsigned int num_filenames;
+  int depth;
+  void** dentries;		/* dirent** */
+  const char* basedir;	/* basedir for @include_dir */
+  unsigned de_max, de_cur;	/* counters into dirent* array */
 };
 
+struct dirent;	/* forward decl */
+
 extern void scanctx_init(struct scan_context *ctx, const char *top_filename);
 extern const char **scanctx_cleanup(struct scan_context *ctx,
                                     unsigned int *num_filenames);
 
+extern const char *scanctx_getpath(struct scan_context *ctx);
+extern const char *scanctx_filename(struct scan_context *ctx, const char *dirname, const char *filename);
+
+extern const char* scanctx_dirnext(struct scan_context* ctx);
+extern int   scanctx_dirscan(struct scan_context* ctx, const char* dirname,
+				int (*filter)(const struct dirent *),
+				int (*compar)(const struct dirent **, const struct dirent **));
+extern int   scanctx_dirend(struct scan_context* ctx);
+extern int   scanctx_inloop(const struct scan_context* ctx);
+
 extern FILE *scanctx_push_include(struct scan_context *ctx, void *prev_buffer,
-                                  const char **error);
+                                  const char *file, const char **error);
 extern void *scanctx_pop_include(struct scan_context *ctx);
 
 #define scanctx_append_string(C, S)             \

lib/scanner.l

@@ -42,6 +42,7 @@
 #include <ctype.h>
 #include <string.h>
 #include <limits.h>
+#include <dirent.h>
 #include "parsectx.h"
 #include "scanctx.h"
 #include "grammar.h"
@@ -83,6 +84,14 @@ static unsigned long long fromhex(const char *s)
 #endif /* __MINGW32__ */
 }
 
+static int filter_dotfiles(const struct dirent *de)
+{
+  const char *fname = de->d_name;
+  return ( NULL != fname 
+    && '\0' != fname[0] /* can't really happen */
+    && '.' != fname[0] ) ? 1 : 0 ;
+}
+
 %}
 
 true              [Tt][Rr][Uu][Ee]
@@ -95,9 +104,10 @@ hex64             0[Xx][0-9A-Fa-f]+L(L)?
 hexchar           \\[Xx][0-9A-Fa-f]{2}
 float             ([-+]?([0-9]*)?\.[0-9]*([eE][-+]?[0-9]+)?)|([-+]?([0-9]+)(\.[0-9]*)?[eE][-+]?[0-9]+)
 comment           (#|\/\/).*$
-include_open      ^[ \t]*@include[ \t]+\"
+include_file_open ^[ \t]*@include[ \t]+\"
+include_dir_open  ^[ \t]*@include_dir[ \t]+\"
 
-%x COMMENT STRING INCLUDE
+%x COMMENT STRING INCLUDE_F INCLUDE_D
 
 %%
 
@@ -126,33 +136,83 @@ include_open      ^[ \t]*@include[ \t]+\"
                     return(TOK_STRING);
                   }
 
-{include_open}    { BEGIN INCLUDE; }
-<INCLUDE>[^\"\\]+ { scanctx_append_string(yyextra, yytext); }
-<INCLUDE>\\\\     { scanctx_append_string(yyextra, "\\"); }
-<INCLUDE>\\\"     { scanctx_append_string(yyextra, "\""); }
-<INCLUDE>\"       {
-                    const char *error;
-                    FILE *fp = scanctx_push_include(yyextra,
-                                                    (void *)YY_CURRENT_BUFFER,
-                                                    &error);
-                    if(fp)
-                    {
-                      yyin = fp;
-                      yy_switch_to_buffer(
-                        yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner),
-                        yyscanner);
-                    }
-                    else
-                    {
-                      yyextra->config->error_text = error;
-                      yyextra->config->error_file = scanctx_current_filename(
-                        yyextra);
-                      yyextra->config->error_line = libconfig_yyget_lineno(
-                        yyscanner);
-                      return TOK_ERROR;
-                    }
-                    BEGIN INITIAL;
-                  }
+{include_file_open}	{ BEGIN INCLUDE_F; }
+<INCLUDE_F>[^\"\\]+	{ scanctx_append_string(yyextra, yytext); }
+<INCLUDE_F>\\\\		{ scanctx_append_string(yyextra, "\\"); }
+<INCLUDE_F>\\\"		{ scanctx_append_string(yyextra, "\""); }
+<INCLUDE_F>\"		{
+        const char *error;
+        FILE *fp = scanctx_push_include(yyextra,
+			(void *)YY_CURRENT_BUFFER,
+			scanctx_getpath(yyextra),
+			&error);
+		if(fp)
+		{
+			yyin = fp;
+			yy_switch_to_buffer(
+				yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner), 
+				yyscanner
+			);
+		}
+		else
+		{
+			yyextra->config->error_text = error;
+			yyextra->config->error_file = scanctx_current_filename(
+				yyextra);
+			yyextra->config->error_line = libconfig_yyget_lineno(
+				yyscanner);
+			return TOK_ERROR;
+		}
+		BEGIN INITIAL;
+	}
+
+{include_dir_open}	{ BEGIN INCLUDE_D; }
+<INCLUDE_D>[^\"\\]+	{ scanctx_append_string(yyextra, yytext); }
+<INCLUDE_D>\\\\		{ scanctx_append_string(yyextra, "\\"); }
+<INCLUDE_D>\\\"		{ scanctx_append_string(yyextra, "\""); }
+<INCLUDE_D>\"		{
+		const char *error;
+		const char* basedir;
+		FILE *fp = NULL;
+
+		basedir = scanctx_getpath(yyextra);
+		if( scanctx_dirscan(yyextra, basedir, filter_dotfiles, alphasort) < 0 )
+		{
+			if(basedir)
+				free((void*)basedir);
+			return TOK_ERROR;
+		}
+
+		if( scanctx_inloop(yyextra) )
+		{
+			fp = scanctx_push_include(yyextra,
+				(void *)YY_CURRENT_BUFFER,
+				scanctx_filename(yyextra, NULL, scanctx_dirnext(yyextra)),
+				&error);
+
+			if(fp)
+			{
+				yyin = fp;
+				yy_switch_to_buffer(
+					yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner), 
+					yyscanner
+				);
+			}
+			else
+			{
+				yyextra->config->error_text = error;
+				yyextra->config->error_file = scanctx_current_filename(
+					yyextra);
+				yyextra->config->error_line = libconfig_yyget_lineno(
+					yyscanner);
+			}
+		}
+		else
+			scanctx_dirend(yyextra);	/* avoid leaks */
+
+		BEGIN INITIAL;
+	}
+
 
 \n|\r|\f          { /* ignore */ }
 [ \t]+            { /* ignore */ }
@@ -194,13 +254,39 @@ include_open      ^[ \t]*@include[ \t]+\"
 .                 { return(TOK_GARBAGE); }
 
 <<EOF>>           {
-                    YY_BUFFER_STATE buf = (YY_BUFFER_STATE)scanctx_pop_include(
-                      yyextra);
-                    if(buf)
-                    {
-                      yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner);
-                      yy_switch_to_buffer(buf, yyscanner);
-                    }
-                    else
-                      yyterminate();
-                  }
+        const char* error;
+	FILE* fp;
+		YY_BUFFER_STATE buf = (YY_BUFFER_STATE)scanctx_pop_include(yyextra);
+		if(buf)
+		{
+			yy_delete_buffer(YY_CURRENT_BUFFER, yyscanner);
+			yy_switch_to_buffer(buf, yyscanner);
+		}
+		else	/* if no more buffers, we are done */
+			yyterminate();
+
+		if( scanctx_inloop(yyextra) )
+		{
+			/* gotta keep looping.... */
+			fp = scanctx_push_include(yyextra,
+					(void *)YY_CURRENT_BUFFER,
+					scanctx_filename(yyextra, NULL, scanctx_dirnext(yyextra)),
+					&error);
+			if(fp)
+			{
+				yyin = fp;
+				yy_switch_to_buffer(
+					yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner), 
+					yyscanner
+				);
+			}
+			else
+			{
+				yyextra->config->error_text = error;
+				yyextra->config->error_file = scanctx_current_filename(yyextra);
+				yyextra->config->error_line = libconfig_yyget_lineno(yyscanner);
+			}
+		}
+		else	/* not on loop, or just finished */
+			scanctx_dirend(yyextra);
+	}