WIP empdump, can't shrink tables

Make.mk

@@ -104,7 +104,7 @@ deps := $(obj:.o=.d)
 # Library archives:
 libs := $(addprefix lib/, libcommon.a libas.a libgen.a libglobal.a)
 # Programs:
-util := $(addprefix src/util/, $(addsuffix $(EXEEXT), empsched fairland files pconfig))
+util := $(addprefix src/util/, $(addsuffix $(EXEEXT), empdump empsched fairland files pconfig))
 client := src/client/empire$(EXEEXT)
 server := src/server/emp_server$(EXEEXT)
 # Info subjects:

doc/xdump

@@ -174,9 +174,10 @@ precisely specified.  We use EBNF (ISO 14977) for syntax, except we
 use '-' in meta-identifiers and omit the concatenation symbol ','.
 
     table = header { record } footer ;
-    header = "XDUMP" space [ "meta" space ] name space timestamp newline ;
+    header = "XDUMP" space [ "meta" space ]
-    name = name-chr { name-chr } ;
+    	     identifier space timestamp newline ;
-    name-chr = ? ASCII characters 33..126 ? ;
+    identifier = id-chr { id-chr } ;
+    id-char = ? ASCII characters 33..126 except '"#()<>=' ? ;
     timestamp = intnum ;
     footer = "/" number newline ;
     record = [ fields ] newline ;
@@ -211,7 +212,7 @@ Notes:
 
 Semantics:
 
-* The table name in the header is one of the names in xdump table.
+* The table identifier in the header is one of the names in xdump table.
 
 * The timestamp increases monotonically.  It has a noticeable
   granularity: game state may change between an xdump and the next
@@ -524,7 +525,7 @@ EBNF changes:
 
 * Header and footer:
 
-    header = "config" name newline colhdr newline ;
+    header = "config" identifier newline colhdr newline ;
     colhdr = { identifier [ "(" ( intnum | identifier ) ")" ] } [ "..." ] ;
     footer = "/config" newline ;
 
@@ -542,7 +543,7 @@ EBNF changes:
     update.
 
   - The column header is due to the self-containedness requirement.
-    It contains just the essential bit of meta-data: the column name.
+    It contains just the essential bit of meta-data: the column names.
 
 * Symbolic fields:
 
@@ -662,7 +663,7 @@ and section `Objectives':
   enemy ships, nations).
 
   (4) Bandwidth will be minimized (i.e. the format will be as
-  concise as possible) while remaining human readable (i.e. no
+  concise as possible) while remaining human-readable (i.e. no
   binary messages).  [Note that data compression may be added at a later
   date, but if it is added, it will be added on a separate port to
   maintain backwards compatability.]
@@ -735,14 +736,14 @@ f. The data to sync is not readily available the server.
    limited view on certain parts of server game state on certain
    events.
 
-   To be complete, a machine readable protocol must disclose as much
+   To be complete, a machine-readable protocol must disclose as much
-   information as the human readable output.  Tracking server game
+   information as the human-readable output.  Tracking server game
    state changes cannot do that alone.  For instance, lookout tells
    you ship#, owner and location.  That event does not trigger any
    state change on the server!
 
-   To be correct, a machine readable protocol must disclose no more
+   To be correct, a machine-readable protocol must disclose no more
-   information than the human readable output.  When you observe a
+   information than the human-readable output.  When you observe a
    server game state change, you can only guess what event triggered
    it, and what it disclosed to which player.  You're stuck with
    conservative assumptions.  That's the death knell for completeness.
@@ -761,7 +762,7 @@ a shortcut, and it didn't work.  That doesn't mean there's no way at
 all.  I believe the only way to get this done right is by tracking
 *events*.  Whenever something is printed to a player, be it live
 connection or telegram, we need to transmit precisely the same
-information in machine readable form.  Much more work.
+information in machine-readable form.  Much more work.
 
 xdump shares valuable ideas with C_SYNC, e.g. using selector
 meta-data.  It is, however, much more modest in scope.  We're pretty

include/xdump.h

@@ -42,11 +42,13 @@
 struct xdstr {
     natid cnum;			/* dump for this country */
     int divine;			/* is this a deity dump? */
+    int human;			/* dump human-readable format */
     void (*pr)(char *fmt, ...);	/* callback for printing dump */
 };
 
-struct xdstr *xdinit(struct xdstr *, natid, void (*)(char *, ...));
+struct xdstr *xdinit(struct xdstr *, natid, int, void (*)(char *, ...));
 extern void xdhdr(struct xdstr *, char *, int);
+extern void xdcolhdr(struct xdstr *, struct castr[]);
 extern void xdflds(struct xdstr *, struct castr[], void *);
 extern struct valstr *xdeval(struct valstr *, struct xdstr *, nsc_type, void *, ptrdiff_t, int, int);
 extern char *xdprval(struct xdstr *, struct valstr *, char *);

man/empdump.6

@@ -0,0 +1,57 @@
+.TH EMPDUMP 6
+.SH NAME
+empdump \- Export/import Empire game state
+.SH SYNOPSIS
+.B empdump
+[
+.B \-mtxhv
+]
+[
+.BI \-e " configfile"
+]
+[
+.I dump-file
+]
+.br
+.SH DESCRIPTION
+.B empdump
+imports and exports game state as plain text.
+.SH OPTIONS
+.TP
+.BI \-e " configfile"
+Use game configuration in \fIconfigfile\fR.
+.TP
+.B \-h
+Help.  Print brief usage information and exit.
+.TP
+.TP
+.B \-m
+Use machine-readable format for export.  Import always recognizes both
+machine-readable and human-readable format.
+.TP
+.B \-t
+Test import, don't update game state.
+.TP
+.B \-v
+Print version information and exit.
+.TP
+.B \-x
+Export game state to standard output.
+.SH OPERANDS
+.TP
+.I dump-file
+The file to import.
+.SH "LIMITATIONS"
+.B empdump
+can't export player bmaps, power report, telegrams, announcements,
+message of the day, no-login message and log files.  Exported
+floating-point values may be inexact.  Importing an exported game
+state may not result in identical data files; besides the loss of
+floating-point precision just mentioned, coordinates are normalized,
+and characters beyond a string's terminating zero in a character array
+may be lost.
+FIXME can't change file size, fix or document here
+.SH "SEE ALSO"
+\fIemp_server\fR(6).
+.SH AUTHOR
+Markus Armbruster <armbru@pond.sub.org>

src/lib/commands/xdump.c

@@ -223,7 +223,7 @@ xdump(void)
     if (!p || !*p)
 	return RET_SYN;
 
-    xdinit(&xd, player->cnum, pr);
+    xdinit(&xd, player->cnum, 0, pr);
     natp = getnatp(player->cnum);
     type = isdigit(p[0]) ? atoi(p) : ef_byname(p);
     if (type >= 0 && type < EF_MAX) {

src/lib/common/xdump.c

@@ -76,20 +76,23 @@
 #include <config.h>
 
 #include <ctype.h>
+#include <limits.h>
 #include "file.h"
 #include "nat.h"
 #include "xdump.h"
 
 /*
  * Initialize XD to dump for country CNUM.
+ * If HUMAN, dump in human-readable format.
  * Dump is to be delivered through callback PR.
  * Return XD.
  */
 struct xdstr *
-xdinit(struct xdstr *xd, natid cnum, void (*pr)(char *fmt, ...))
+xdinit(struct xdstr *xd, natid cnum, int human, void (*pr)(char *fmt, ...))
 {
     xd->cnum = cnum;
     xd->divine = getnatp(cnum)->nat_stat == STAT_GOD;
+    xd->human = human;
     xd->pr = pr;
     return xd;
 }
@@ -116,25 +119,17 @@ xdeval(struct valstr *val, struct xdstr *xd,
     return val;			/* FIXME nstr_exec_val() should return VAL */
 }
 
-/* Dump VAL prefixed with SEP, return " ".  */
+/*
-char *
+ * Dump string STR to XD with funny characters escaped.
-xdprval(struct xdstr *xd, struct valstr *val, char *sep)
+ * Dump at most N characters.
+ */
+static void
+xdpresc(struct xdstr *xd, char *str, size_t n)
 {
     unsigned char *s, *e, *l;
 
-    switch (val->val_type) {
+    s = (unsigned char *)str;
-    case NSC_LONG:
+    l = s + n;
-	xd->pr("%s%ld", sep, val->val_as.lng);
-	break;
-    case NSC_DOUBLE:
-	xd->pr("%s%#g", sep, val->val_as.dbl);
-	break;
-    case NSC_STRING:
-	s = (unsigned char *)val->val_as.str.base;
-	if (s) {
-	    xd->pr("%s\"", sep);
-	    l = s + val->val_as.str.maxsz;
-	    /* FIXME maxsz == INT_MAX ! */
     for (;;) {
 	for (e = s;
 	     e < l && *e != '"' && *e != '\\' && isgraph(*e);
@@ -147,6 +142,27 @@ xdprval(struct xdstr *xd, struct valstr *val, char *sep)
 	    break;
 	s = e;
     }
+}
+
+/* Dump VAL in machine readable format, prefixed with SEP, return " ".  */
+static char *
+xdprval_nosym(struct xdstr *xd, struct valstr *val, char *sep)
+{
+    switch (val->val_type) {
+    case NSC_LONG:
+	xd->pr("%s%ld", sep, val->val_as.lng);
+	break;
+    case NSC_DOUBLE:
+#if 0 /* TODO have %a */
+	xd->pr("%s%a", sep, val->val_as.dbl);
+#else
+	xd->pr("%s%#g", sep, val->val_as.dbl);
+#endif
+	break;
+    case NSC_STRING:
+	if (val->val_as.str.base) {
+	    xd->pr("%s\"", sep);
+	    xdpresc(xd, val->val_as.str.base, val->val_as.str.maxsz);
 	    xd->pr("\"");
 	} else
 	    xd->pr("%snil", sep);
@@ -158,6 +174,66 @@ xdprval(struct xdstr *xd, struct valstr *val, char *sep)
     return " ";
 }
 
+/*
+ * Dump symbol with value KEY from symbol table TYPE to XD.
+ * Prefix with SEP, return " ".
+ */
+static char *
+xdprsym(struct xdstr *xd, int key, int type, char *sep)
+{
+    char *sym = symbol_by_value(key, ef_ptr(type, 0));
+
+    if (CANT_HAPPEN(!sym))
+	xd->pr("%s%ld", sep, key);
+    else {
+	xd->pr("%s", sep);
+	xdpresc(xd, sym, INT_MAX);
+    }
+    return " ";
+}
+
+/*
+ * Dump VAL prefixed with SEP, return " ".
+ * CA describes the field from which the value was fetched.
+ */
+static char *
+xdprval_sym(struct xdstr *xd, struct valstr *val, struct castr *ca, char *sep)
+{
+    unsigned long bit;
+    struct castr *ca_sym;
+    char *sym;
+
+    if (xd->human && val->val_type == NSC_LONG && ca->ca_table != EF_BAD) {
+	ca_sym = ef_cadef(ca->ca_table);
+	if (ca_sym != symbol_ca)
+	    ;
+	else if (ca->ca_flags & NSC_BITS) {
+	    xd->pr("%s(", sep);
+	    sep = "";
+	    for (bit = 1; bit; bit <<= 1) {
+		if (bit & val->val_as.lng)
+		    sep = xdprsym(xd, bit, ca->ca_table, sep);
+	    }
+	    xd->pr(")");
+	    return " ";
+	} else
+	    return xdprsym(xd, val->val_as.lng, ca->ca_table, sep);
+    }
+
+    return xdprval_nosym(xd, val, sep);
+}
+
+/*
+ * Dump VAL prefixed with SEP, return " ".
+ * XD must not be human-readable.
+ */
+char *
+xdprval(struct xdstr *xd, struct valstr *val, char *sep)
+{
+    CANT_HAPPEN(xd->human);
+    return xdprval_nosym(xd, val, sep);
+}
+
 /*
  * Dump field values of a context object to XD.
  * CA[] describes fields.
@@ -180,25 +256,65 @@ xdflds(struct xdstr *xd, struct castr ca[], void *ptr)
 	do {
 	    xdeval(&val, xd,
 		   ca[i].ca_type, ptr, ca[i].ca_off, j, ca[i].ca_len);
-	    sep = xdprval(xd, &val, sep);
+	    sep = xdprval_sym(xd, &val, &ca[i], sep);
 	} while (++j < n);
     }
 }
 
 /*
- * Dump header for dump NAME.
+ * Dump header for dump NAME to XD.
  * If META, it's for the meta-data dump rather than the data dump.
  */
 void
 xdhdr(struct xdstr *xd, char *name, int meta)
 {
-    xd->pr("XDUMP %s%s %ld\n", meta ? "meta " : "", name, (long)time(NULL));
+    if (xd->human) {
+	xd->pr("config %s\n", name);
+    } else
+	xd->pr("XDUMP %s%s %ld\n",
+	       meta ? "meta " : "", name, (long)time(NULL));
 }
 
-/* Dump footer for a dump that dumped N objects.  */
+/*
+ * Dump column header to XD.
+ * CA[] describes fields.
+ * Does nothing unless XD is human-readable.
+ */
+void
+xdcolhdr(struct xdstr *xd, struct castr ca[])
+{
+    int i, j, n;
+    char *sep = "";
+
+    if (!xd->human)
+	return;
+
+    for (i = 0; ca[i].ca_name; ++i) {
+	if (ca[i].ca_flags & NSC_DEITY && !xd->divine)
+	    continue;
+	if (ca[i].ca_flags & NSC_EXTRA)
+	    continue;
+	n = ca[i].ca_type != NSC_STRINGY ? ca[i].ca_len : 0;
+	if (n) {
+	    for (j = 0; j < n; j++) {
+		xd->pr("%s%s(%d)",sep, ca[i].ca_name, j);
+		sep = " ";
+	    }
+	} else {
+	    xd->pr("%s%s", sep, ca[i].ca_name);
+	    sep = " ";
+	}
+    }
+    xd->pr("\n");
+}
+
+/* Dump footer for a dump that dumped N objects to XD.  */
 void
 xdftr(struct xdstr *xd, int n)
 {
+    if (xd->human)
+	xd->pr("/config\n");
+    else
 	xd->pr("/%d\n", n);
 }
 
@@ -217,6 +333,7 @@ xdmeta(struct xdstr *xd, int type)
 	return RET_SYN;
 
     xdhdr(xd, ef_nameof(type), 1);
+    xdcolhdr(xd, ca);
 
     for (i = 0; ca[i].ca_name; i++) {
 	if (ca[i].ca_flags & NSC_DEITY && !xd->divine)

src/lib/common/xundump.c

@@ -132,21 +132,6 @@ skipfs(FILE *fp)
     return ch;
 }
 
-/*
- * Read an identifier from FP into BUF.
- * BUF must have space for 1024 characters.
- * Return number of characters read on success, -1 on failure.
- */
-static int
-getid(FILE *fp, char *buf)
-{
-    int n;
-    if (fscanf(fp, "%1023[^#()<>=#\" \t\n]%n", buf, &n) != 1
-	|| !isalpha(buf[0]))
-	return -1;
-    return n;
-}
-
 /*
  * Decode escape sequences in BUF.
  * Return BUF on success, null pointer on failure.
@@ -172,6 +157,22 @@ xuesc(char *buf)
     return buf;
 }
 
+/*
+ * Read an identifier from FP into BUF.
+ * BUF must have space for 1024 characters.
+ * Return number of characters read on success, -1 on failure.
+ */
+static int
+getid(FILE *fp, char *buf)
+{
+    int n;
+    if (fscanf(fp, "%1023[^\"#()<>= \t\n]%n", buf, &n) != 1
+	|| !isalpha(buf[0]))
+	return -1;
+    xuesc(buf);
+    return n;
+}
+
 /*
  * Try to read a field name from FP.
  * I is the field number, counting from zero.

src/util/empdump.c

@@ -0,0 +1,200 @@
+/*
+ *  Empire - A multi-player, client/server Internet based war game.
+ *  Copyright (C) 1986-2008, Dave Pare, Jeff Bailey, Thomas Ruschak,
+ *                           Ken Stevens, Steve McClure
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  ---
+ *
+ *  See files README, COPYING and CREDITS in the root of the source
+ *  tree for related information and legal notices.  It is expected
+ *  that future projects/authors will amend these files as needed.
+ *
+ *  ---
+ *
+ *  empdump.c: Export/import Empire game state
+ * 
+ *  Known contributors to this file:
+ *     Markus Armbruster, 2008
+ */
+
+#include <config.h>
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <unistd.h>
+#include "file.h"
+#include "nat.h"
+#include "optlist.h"
+#include "prototypes.h"
+#include "version.h"
+#include "xdump.h"
+
+static void dump_table(int, int);
+
+int
+main(int argc, char *argv[])
+{
+    char *config_file = NULL;
+    char *import = NULL;
+    int export = 0;
+    int private = 0;
+    int human = 1;
+    int opt, i, lineno, type;
+    FILE *impf;
+    int dirty[EF_MAX];
+
+    while ((opt = getopt(argc, argv, "e:mtxhv")) != EOF) {
+	switch (opt) {
+	case 'e':
+	    config_file = optarg;
+	    break;
+	case 'h':
+	    printf("Usage: %s [OPTION]... [DUMP-FILE]\n"
+		   "  -e CONFIG-FILE  configuration file\n"
+		   "                  (default %s)\n"
+		   "  -m              use machine-readable format\n"
+		   "  -t              test import, don't update game state\n"
+		   "  -x              export to standard output\n"
+		   "  -h              display this help and exit\n"
+		   "  -v              display version information and exit\n",
+		   argv[0], dflt_econfig);
+	    exit(0);
+	case 'm':
+	    human = 0;
+	    break;
+	case 't':
+	    private = EFF_PRIVATE;
+	    break;
+	case 'x':
+	    export = 1;
+	    break;
+	case 'v':
+	    printf("%s\n\n%s", version, legal);
+	    exit(0);
+	default:
+	    fprintf(stderr, "Try -h for help.\n");
+	    exit(1);
+	}
+    }
+
+    if (argv[optind])
+	import = argv[optind++];
+
+    if (import) {
+	impf = fopen(import, "r");
+	if (!impf) {
+	    fprintf(stderr, "Cant open %s for reading (%s)\n",
+		    import, strerror(errno));
+	    exit(1);
+	}
+    } else
+	private = EFF_PRIVATE;
+
+    /* read configuration & initialize */
+    empfile_init();
+    if (emp_config(config_file) < 0)
+	exit(1);
+    empfile_fixup();
+    nsc_init();
+    if (read_builtin_tables() < 0)
+	exit(1);
+    if (read_custom_tables() < 0)
+	exit(1);
+    if (chdir(gamedir)) {
+	fprintf(stderr, "Can't chdir to %s (%s)\n",
+		gamedir, strerror(errno));
+	exit(1);
+    }
+    global_init();
+
+    for (i = 0; i < EF_MAX; i++) {
+	if (!EF_IS_GAME_STATE(i))
+	    continue;
+	if (!ef_open(i, EFF_MEM | private))
+	    exit(1);
+    }
+
+    /* import from IMPORT */
+    memset(dirty, 0, sizeof(dirty));
+    if (import) {
+	lineno = 1;
+	while ((type = xundump(impf, import, &lineno, EF_BAD)) >= 0)
+	    dirty[type] = 1;
+	if (type == EF_BAD)
+	    exit(1);
+    }
+
+    if (ef_verify() < 0)
+	exit(1);
+
+    /* export to stdout */
+    if (export) {
+	for (i = 0; i < EF_MAX; i++) {
+	    if (!EF_IS_GAME_STATE(i))
+		continue;
+	    dump_table(i, human);
+	}
+    }
+
+    /* write out imported data */
+    for (i = 0; i < EF_MAX; i++) {
+	if (!EF_IS_GAME_STATE(i))
+	    continue;
+	if (!private && dirty[i]) {
+	    if (!ef_close(i))
+		exit(1);
+	}
+    }
+
+    return 0;
+}
+
+static void
+printf_wrapper(char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    vprintf(fmt, ap);
+    va_end(ap);
+}
+
+static void
+dump_table(int type, int human)
+{
+    struct xdstr xd;
+    struct castr *ca;
+    int i;
+    void *p;
+
+    ca = ef_cadef(type);
+    if (!ca)
+	return;
+
+    xdinit(&xd, 0, human, printf_wrapper);
+    xdhdr(&xd, ef_nameof(type), 0);
+    xdcolhdr(&xd, ca);
+    for (i = 0; (p = ef_ptr(type, i)); i++) {
+	xdflds(&xd, ca, p);
+	printf("\n");
+    }
+    xdftr(&xd, i);
+}