--- /dev/null
+/* reservoir_sample.c
+ This generates a randomized subset of its input, by means of reservoir-
+ sampling, and a Fisher-Yates shuffle.
+*/
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <sys/uio.h>
+#include <time.h>
+#include <sysexits.h>
+#include <assert.h>
+
+#include "version.h"
+#include "notify.h"
+#include "test_suite.h"
+
+static const char * const src_id_ = "v" VERSION_STR " " VERSION_DATE;
+
+static struct options_ {
+ unsigned int verbosity;
+ size_t read_buf_sz;
+ size_t reservoir_sz;
+ char delim;
+ char delim_out;
+ char *rand_file;
+} options_ = {
+ .verbosity = 0,
+ .read_buf_sz = 8192,
+ .reservoir_sz = 1,
+ .delim = '\n',
+ .delim_out = '\n',
+ .rand_file = "/dev/random",
+};
+
+static int rand_fd_ = -1;
+
+typedef struct buf_ {
+ size_t buf_sz;
+ size_t buf_start;
+ size_t buf_used;
+ unsigned char buf[];
+} *buf_t;
+#define BUF_ROOM(__b__) ( (__b__)->buf_sz - ( (__b__)->buf_start + (__b__)->buf_used ) )
+
+typedef struct reservoir_ {
+ size_t reservoir_sz;
+ buf_t reservoir[];
+} *reservoir_t;
+
+#ifndef NDEBUG
+#define D_BUF(__pre__,__b__,...) do {\
+ if ( (__b__) == NULL )\
+ NOTIFY_DEBUG(__pre__ "buf:%p", ## __VA_ARGS__, (__b__));\
+ else {\
+ NOTIFY_DEBUG(__pre__ "buf:%p sz:%zu start:%zu used:%zu free:%zu '%.*s'",\
+ ## __VA_ARGS__,\
+ (__b__),\
+ (__b__)->buf_sz,\
+ (__b__)->buf_start,\
+ (__b__)->buf_used,\
+ BUF_ROOM((__b__)),\
+ (int)(__b__)->buf_used, (__b__)->buf + (__b__)->buf_start);\
+ assert( (__b__)->buf_sz >= (__b__)->buf_start + (__b__)->buf_used );\
+ }\
+} while (0)
+#define D_RESERVOIR(__r__) do {\
+ size_t i;\
+ for (i = 0; i < (__r__)->reservoir_sz; i++) {\
+ D_BUF("reservoir[%zu] ", (__r__)->reservoir[i], i); } } while (0)
+#else
+#define D_RESERVOIR(...)
+#define D_BUF(...)
+#endif
+
+
+static
+void usage_(const char *prog, unsigned int full) {
+ FILE *f = full ? stdout : stderr;
+ char *x = strrchr(prog, '/');
+
+ if (x && *(x + 1))
+ prog = x + 1;
+
+ if (full)
+ fprintf(f, "%s -- returns a random sampling of input\n\n",
+ prog);
+
+ fprintf(f, "Usage: %s options\n",
+ prog);
+
+ if (full) {
+ fprintf(f, "\nOptions:\n"
+ "\t-n <num> -- returns <num> samples [default: %zu]\n"
+ "\t-d <delim> -- use <delim> as input delimiter [default: '\\%03hho']\n"
+ "\t-r <file> -- read randomness from <file> [default: '%s']\n"
+ "\t-b <bytes> -- read buffer size [default: %zu]\n"
+ "\t-v -- increase verbosity\n"
+ "\t-h -- this screen\n",
+ options_.reservoir_sz,
+ options_.delim,
+ options_.rand_file,
+ options_.read_buf_sz);
+
+ fprintf(f, "\n%78s\n", src_id_);
+ }
+ fflush(f);
+}
+
+
+/* rand_upto_inclusive_
+ Room for improvement: constrain bits of randomness consumed, based on #limit
+ also maybe read chunks of randomness at a time
+ */
+static
+unsigned long rand_upto_inclusive_(unsigned long limit) {
+ unsigned long randomness;
+
+ if (limit == 0)
+ return 0;
+
+ if (rand_fd_ != -1) {
+ ssize_t len;
+
+ len = read(rand_fd_, &randomness, sizeof randomness);
+ if (len == sizeof randomness) {
+ randomness %= limit + 1;
+
+ return randomness;
+ }
+ NOTIFY_ERROR("%s(%d, %zu):%zd:%s", "read", rand_fd_, sizeof randomness, len, (len < 0) ? strerror(errno) : ( (len == 0) ? "EOF" : "not enough read consecutively") );
+ }
+
+ randomness = mrand48();
+ randomness %= limit + 1;
+
+ return randomness;
+}
+
+
+static
+int rand_init_(char *rand_file) {
+ srand48(time(NULL) ^ getpid());
+ if (rand_file) {
+ rand_fd_ = open(rand_file, O_RDONLY);
+ if (rand_fd_ == -1) {
+ NOTIFY_ERROR("%s('%s'):%s", "open", rand_file, strerror(errno));
+ return -1;
+ }
+ }
+ return 0;
+}
+
+
+static
+buf_t buf_new_(size_t sz) {
+ buf_t buf = malloc(sz + sizeof *buf);
+ if (buf) {
+ buf->buf_sz = sz;
+ buf->buf_start = buf->buf_used = 0;
+ memset(buf->buf, 0, sz);
+ }
+ return buf;
+}
+
+
+static
+void buf_rebase_(buf_t buf) {
+ if (buf->buf_start == 0)
+ return;
+ memmove(buf->buf, buf->buf + buf->buf_start, buf->buf_used);
+ buf->buf_start = 0;
+}
+
+
+static
+int buf_makeroom_(buf_t *pbuf, size_t roomfor) {
+ size_t new_sz;
+ void *tmp_ptr;
+
+ assert(pbuf != NULL);
+
+ if (*pbuf == NULL) {
+ *pbuf = buf_new_(roomfor);
+ if (*pbuf == NULL) {
+ return -1;
+ }
+ }
+
+ buf_rebase_(*pbuf);
+
+ if (BUF_ROOM(*pbuf) >= roomfor)
+ return 0;
+
+ new_sz = (*pbuf)->buf_used + roomfor;
+ tmp_ptr = realloc(*pbuf, new_sz + sizeof **pbuf);
+ if (tmp_ptr == NULL) {
+ NOTIFY_ERROR("%s:%s", "realloc", strerror(errno));
+ return -1;
+ }
+ *pbuf = tmp_ptr;
+ (*pbuf)->buf_sz = new_sz;
+
+ return 0;
+}
+
+
+static
+int buf_range_dup_or_append_(buf_t src, size_t src_skip, size_t n, buf_t *dst) {
+ assert(src != NULL);
+ assert(dst != NULL);
+ assert(src_skip + n <= src->buf_used);
+
+ if (buf_makeroom_(dst, n)) {
+ return -1;
+ }
+
+ memcpy((*dst)->buf + (*dst)->buf_used, src->buf + src->buf_start + src_skip, n);
+ (*dst)->buf_used += n;
+
+ return 0;
+}
+
+
+/* buf_flense_
+ Starting after #src_offset characters, scan through #src, stopping at
+ the first character matching #delimiter, whereupon all the characters
+ leading up to #delimiter are copied into *#dst if #dst is not NULL. #src
+ becomes the characters following #delimiter.
+ Returns the number of characters flensed from #src.
+
+ Room for improvement:
+ If flensed segment is more than half the buffer, copy remainder of src
+ into dst, then return src, leaving dst in its place.
+*/
+static
+ssize_t buf_flense_(buf_t *src, size_t src_offset, int delimiter, buf_t *dst) {
+ const size_t delimiter_len = 1;
+ size_t i;
+
+ assert(src != NULL);
+ assert(src_offset <= (*src)->buf_used);
+
+ NOTIFY_DEBUG("src_offset:%zu", src_offset);
+ D_BUF("src ", *src);
+ D_BUF("dst ", dst ? *dst : NULL);
+
+ for (i = src_offset; i < (*src)->buf_used; i++) {
+ if ((*src)->buf[(*src)->buf_start + i] == delimiter) {
+
+ if (dst != NULL) {
+ if (buf_range_dup_or_append_((*src), 0, i, dst)) {
+ return -1;
+ }
+ }
+
+ (*src)->buf_start += i + delimiter_len;
+ (*src)->buf_used -= i + delimiter_len;
+
+ D_BUF("src ", *src);
+ D_BUF("dst ", dst ? *dst : NULL);
+ return i + delimiter_len;
+ }
+ }
+
+ return 0;
+}
+
+#ifdef TEST
+
+static const char buf_flense_testdata1[] = "a\nbc\ndef\nghij\nklmno\npqr\ns\ntuv\nwxyz0\n1234\n567\n89\n0leftovers";
+static const char buf_flense_testdata2[] = "\n\nfoo\nbar\n\n";
+
+struct buf_flense_expected_result_ {
+ ssize_t r;
+ const char *buf;
+} buf_flense_expected1[] = {
+ { 1 + 1, "a" },
+ { 2 + 1, "bc" },
+ { 3 + 1, "def" },
+ { 4 + 1, "ghij" },
+ { 5 + 1, "klmno" },
+ { 3 + 1, "pqr" },
+ { 1 + 1, "s" },
+ { 3 + 1, "tuv" },
+ { 5 + 1, "wxyz0" },
+ { 4 + 1, "1234" },
+ { 3 + 1, "567" },
+ { 2 + 1, "89" },
+ { 0, "0leftovers" },
+}, buf_flense_expected2[] = {
+ { 0 + 1, "" },
+ { 0 + 1, "" },
+ { 3 + 1, "foo" },
+ { 3 + 1, "bar" },
+ { 0 + 1, "" },
+ { 0, "" },
+};
+
+struct test_buf_flense_data_ {
+ const char *src;
+ const struct buf_flense_expected_result_ *expected;
+} test_buf_flense_data1 = {
+ .src = buf_flense_testdata1,
+ .expected = buf_flense_expected1,
+}, test_buf_flense_data2 = {
+ .src = buf_flense_testdata2,
+ .expected = buf_flense_expected2,
+};
+
+static int test_buf_flense_(void *test_arg, void *suite_arg) {
+ (void)suite_arg;
+ struct test_buf_flense_data_ *test_data = (struct test_buf_flense_data_ *)test_arg;
+ const char testdata_len = strlen(test_data->src);
+ const struct buf_flense_expected_result_ *next_result = test_data->expected;
+ int retval = 0;
+ buf_t src;
+
+ TEST_INFO("initializing src buffer with %zu characters", testdata_len);
+ src = buf_new_(testdata_len);
+ if (src == NULL) {
+ TEST_ERROR("%s:%s", "new_buf_", "failed");
+ return -1;
+ }
+ memcpy(src->buf, test_data->src, testdata_len);
+ src->buf_used += testdata_len;
+
+ D_BUF("src ", src);
+
+ for (;;) {
+ ssize_t r;
+ buf_t dst = NULL;
+
+ r = buf_flense_(&src, 0, '\n', &dst);
+ if (r != next_result->r) {
+ TEST_ERROR("result '%zd' does not match expected result '%zd'", r, next_result->r);
+ retval = -1;
+ }
+ if (r == 0) {
+ TEST_INFO("finished flensing");
+ break;
+ }
+
+ if (strlen(next_result->buf) > dst->buf_used) {
+ TEST_ERROR("flensed buffer smaller than expected result '%s'", next_result->buf);
+ D_BUF("dst ", dst);
+ retval = -1;
+ } else if (memcmp(next_result->buf, dst->buf + dst->buf_start, strlen(next_result->buf))) {
+ TEST_ERROR("flensed buffer does not match expected result '%s'", next_result->buf);
+ D_BUF("dst ", dst);
+ retval = -1;
+ }
+
+ TEST_INFO("flensed: '%.*s'", (int)dst->buf_used, dst->buf + dst->buf_start);
+
+ memset(dst, 0, dst->buf_sz + sizeof *dst);
+ free(dst);
+ dst = NULL;
+
+ next_result++;
+ }
+ if (strlen(next_result->buf) > src->buf_used) {
+ TEST_ERROR("remaining buffer smaller than expected result '%s'", next_result->buf);
+ D_BUF("src ", src);
+ retval = -1;
+ } else if (memcmp(next_result->buf, src->buf + src->buf_start, strlen(next_result->buf))) {
+ TEST_ERROR("remaining buffer does not match expected result '%s'", next_result->buf);
+ D_BUF("src ", src);
+ retval = -1;
+ }
+
+ TEST_INFO("remains: '%.*s'", (int)src->buf_used, src->buf + src->buf_start);
+
+ memset(src, 0, src->buf_sz + sizeof *src);
+ free(src);
+ src = NULL;
+
+ return retval;
+}
+#endif /* TEST */
+
+
+/* reservoir_remember_
+ Remember #line, forgetting a line if more than #num_lines have already
+ been remembered. Remembers them in random order.
+*/
+static
+void reservoir_remember_(struct reservoir_ *reservoir, size_t num_lines, buf_t buf) {
+ unsigned long randomness;
+ buf_t old_buf;
+
+ assert(reservoir != NULL);
+
+ D_BUF("reserving ", buf);
+
+ if (reservoir->reservoir_sz > 0) {
+ if (num_lines < reservoir->reservoir_sz) {
+ randomness = rand_upto_inclusive_(num_lines);
+ NOTIFY_DEBUG("moving index %zu to end (%zu)", randomness, num_lines);
+ reservoir->reservoir[num_lines] = reservoir->reservoir[randomness];
+ old_buf = NULL;
+ } else {
+ randomness = rand_upto_inclusive_(reservoir->reservoir_sz - 1);
+ old_buf = reservoir->reservoir[randomness];
+ }
+ NOTIFY_DEBUG("replacing reservoir index %zu", randomness);
+ reservoir->reservoir[randomness] = buf;
+ } else {
+ old_buf = buf;
+ }
+
+ if (old_buf != NULL) {
+ D_BUF("FREEING ", old_buf);
+ memset(old_buf, 0, old_buf->buf_sz + sizeof *old_buf);
+ free(old_buf);
+ }
+
+ D_RESERVOIR(reservoir);
+}
+
+
+/* reservoir_read_
+ Read (up to #read_block_sz bytes at a time) from #fd (until EOF) into an
+ accumulator buffer. For each #delimiter character found in what was just
+ read, occasionally remember the preceeding characters.
+*/
+static
+int reservoir_read_(int fd, size_t read_block_sz, int delimiter, size_t *num_lines, struct reservoir_ *reservoir) {
+ buf_t read_buf, new_buf = NULL;
+ size_t line_scanned; /* how much of the buffer has already been searched for delimiter */
+ ssize_t len;
+
+ assert(read_block_sz > 0);
+ assert(num_lines != NULL);
+ assert(reservoir != NULL);
+
+ if (fd < 0) {
+ return -1;
+ }
+
+ read_buf = buf_new_(read_block_sz);
+ if (read_buf == NULL) {
+ return -1;
+ }
+ line_scanned = 0;
+
+ for (;;) {
+ NOTIFY_DEBUG("read loop\n\n");
+
+ if (buf_makeroom_(&read_buf, read_block_sz)) {
+ free(read_buf);
+ free(new_buf);
+ return -1;
+ }
+
+ NOTIFY_DEBUG("read(%d, %p + %zu + %zu, %zu)", fd, read_buf->buf, read_buf->buf_start, line_scanned, BUF_ROOM(read_buf));
+ len = read(fd, read_buf->buf + read_buf->buf_start + line_scanned, BUF_ROOM(read_buf));
+ if (len < 0) {
+ NOTIFY_ERROR("%s:%s", "read", strerror(errno));
+ free(read_buf);
+ free(new_buf);
+ return -1;
+ }
+ if (len == 0) {
+ break;
+ }
+ read_buf->buf_used += len;
+
+ NOTIFY_DEBUG("len:%zd", len);
+ D_BUF("read_buf: ", read_buf);
+
+ while (len > 0) {
+ ssize_t len_flensed;
+
+ NOTIFY_DEBUG("len:%zd", len);
+
+ /* determine if we want to save the next buffer */
+ if (new_buf == NULL) {
+ /* if new_buf is not null, we already want to save the next one.. */
+ /* otherwise, save if we've read in fewer lines than the reservoir holds */
+ /* or else there's a reservoir_sz-in-num_lines chance of saving the next line */
+
+ if (*num_lines < reservoir->reservoir_sz) {
+ NOTIFY_DEBUG("still filling reservoir.. %zu filled out of %zu..", *num_lines, reservoir->reservoir_sz);
+ } else {
+ NOTIFY_DEBUG("will save if random [0-%zu] is less than %zu...", *num_lines, reservoir->reservoir_sz);
+ }
+
+ if (*num_lines < reservoir->reservoir_sz
+ || rand_upto_inclusive_(*num_lines) < reservoir->reservoir_sz) {
+ NOTIFY_DEBUG("next buffer will be reserved..");
+ new_buf = buf_new_(0);
+ if (new_buf == NULL) {
+ free(read_buf);
+ return -1;
+ }
+ } else {
+ NOTIFY_DEBUG("not saving next buffer..");
+ }
+ }
+
+ len_flensed = buf_flense_(&read_buf, line_scanned, delimiter, new_buf ? &new_buf : NULL);
+ if (len_flensed < 0) {
+ free(read_buf);
+ free(new_buf);
+ return -1;
+ }
+ if (len_flensed == 0) {
+ /* no delimiter found yet, stop parsing and read more */
+ NOTIFY_DEBUG("no delim found after %zd", len);
+ line_scanned = len;
+ buf_rebase_(read_buf);
+ break;
+ }
+
+ len -= len_flensed;
+ line_scanned = 0;
+
+ D_BUF("read_buf: ", read_buf);
+
+ if (new_buf != NULL) {
+ D_BUF("parsed complete line: ", new_buf);
+ reservoir_remember_(reservoir, *num_lines, new_buf);
+ new_buf = NULL;
+ *num_lines += 1;
+ }
+
+ }
+ }
+ /* leftovers */
+ NOTIFY_DEBUG("loop done\n\n");
+ if (read_buf->buf_used) {
+ D_BUF("leftovers: ", read_buf);
+
+ if (new_buf != NULL
+ || *num_lines < reservoir->reservoir_sz) {
+ reservoir_remember_(reservoir, *num_lines, read_buf);
+ read_buf = NULL;
+ }
+
+ *num_lines += 1;
+ }
+
+ free(read_buf);
+ free(new_buf);
+
+ return 0;
+}
+
+
+/* reservoir_write_
+ Room for improvement: unroll reservoir into (up to IOV_MAX) more iov slots.
+*/
+static
+int reservoir_write_(int fd, struct reservoir_ *reservoir, char delim) {
+ ssize_t len;
+ size_t i;
+ struct iovec iov[2];
+
+ iov[1].iov_base = &delim;
+ iov[1].iov_len = sizeof delim;
+
+ assert(reservoir != NULL);
+ D_RESERVOIR(reservoir);
+
+ for (i = 0; i < reservoir->reservoir_sz; i++) {
+ if (reservoir->reservoir[i]) {
+ iov[0].iov_base = reservoir->reservoir[i]->buf + reservoir->reservoir[i]->buf_start;
+ iov[0].iov_len = reservoir->reservoir[i]->buf_used;
+ } else {
+ iov[0].iov_base = NULL;
+ iov[0].iov_len = 0;
+ }
+
+ len = writev(fd, iov, sizeof iov / sizeof *iov);
+ if (len < 0) {
+ NOTIFY_ERROR("%s:%s", "writev", strerror(errno));
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+#ifndef TEST
+
+int main(int argc, char *argv[]) {
+ struct reservoir_ *reservoir;
+ size_t num_lines = 0;
+ int fd;
+ int c;
+
+ while ( (c = getopt(argc, argv, "hvb:n:d:")) != EOF ) {
+ switch (c) {
+ case 'v':
+ options_.verbosity++;
+ break;
+
+ case 'b':
+ options_.read_buf_sz = atoi(optarg);
+ break;
+
+ case 'd':
+ options_.delim = *optarg;
+ options_.delim_out = *optarg;
+ break;
+
+ case 'n':
+ options_.reservoir_sz = atoi(optarg);
+ break;
+
+ case 'h':
+ usage_(argv[0], 1);
+ exit(EX_OK);
+
+ default:
+ usage_(argv[0], 0);
+ exit(EX_USAGE);
+ }
+ }
+
+ if (argc - optind < 0) {
+ usage_(argv[0], 0);
+ exit(EX_USAGE);
+ }
+
+ if (rand_init_(options_.rand_file)) {
+ NOTIFY_ERROR("failed to initialize randomness source\n");
+ exit(EX_NOINPUT);
+ }
+
+ reservoir = malloc((options_.reservoir_sz * sizeof *reservoir->reservoir) + sizeof *reservoir);
+ if (reservoir == NULL) {
+ NOTIFY_ERROR("%s:%s", "calloc", strerror(errno));
+ exit(EX_OSERR);
+ }
+ reservoir->reservoir_sz = options_.reservoir_sz;
+ memset(reservoir->reservoir, 0, reservoir->reservoir_sz * sizeof *reservoir->reservoir);
+
+ if (argc - optind == 0) {
+ fd = STDIN_FILENO;
+ if (reservoir_read_(fd, options_.read_buf_sz, options_.delim, &num_lines, reservoir)) {
+ exit(EX_SOFTWARE);
+ }
+ } else {
+ while (optind < argc) {
+ if (strcmp("-", argv[optind]) == 0) {
+ fd = STDIN_FILENO;
+ if (reservoir_read_(fd, options_.read_buf_sz, options_.delim, &num_lines, reservoir)) {
+ exit(EX_SOFTWARE);
+ }
+ } else {
+ fd = open(argv[optind], O_RDONLY);
+ if (fd < 0) {
+ NOTIFY_ERROR("%s('%s'):%s", "open", argv[optind], strerror(errno));
+ exit(EX_NOINPUT);
+ }
+ if (reservoir_read_(fd, options_.read_buf_sz, options_.delim, &num_lines, reservoir)) {
+ exit(EX_SOFTWARE);
+ }
+ if (close(fd)) {
+ NOTIFY_ERROR("%s:%s", "close", strerror(errno));
+ exit(EX_OSERR);
+ }
+ }
+ optind++;
+ }
+ }
+
+ if (options_.verbosity) {
+ fprintf(stderr, "%zu sample%s, out of %zu total choices\n", options_.reservoir_sz, options_.reservoir_sz > 1 ? "s" : "", num_lines);
+ }
+
+ if (reservoir_write_(STDOUT_FILENO, reservoir, options_.delim_out)) {
+ exit(EX_SOFTWARE);
+ }
+
+ exit(EX_OK);
+}
+
+#else /* TEST */
+
+void *test_suite_data;
+
+int test_suite_pre(void *suite_data) {
+ (void)suite_data;
+ if (rand_init_(NULL)) {
+ return -1;
+ }
+ return 0;
+}
+int test_suite_post(void *suite_data) {
+ (void)suite_data;
+ return 0;
+}
+
+test_t test_suite[] = {
+ { "test_buf_flense_ 1", test_buf_flense_, NULL, NULL, &test_buf_flense_data1 },
+ { "test_buf_flense_ 2", test_buf_flense_, NULL, NULL, &test_buf_flense_data2 },
+ { NULL, NULL, NULL, NULL, NULL },
+};
+
+
+#endif /* TEST */
projects / reservoir_sample / commitdiff