monotone

monotone Mtn Source Tree

Root/database.cc

1// -*- mode: C++; c-file-style: "gnu"; indent-tabs-mode: nil -*-
2// copyright (C) 2002, 2003 graydon hoare <graydon@pobox.com>
3// all rights reserved.
4// licensed to the public under the terms of the GNU GPL (>= 2)
5// see the file COPYING for details
6
7#include <fstream>
8#include <iterator>
9#include <list>
10#include <set>
11#include <sstream>
12#include <vector>
13
14#include <stdarg.h>
15
16#include <boost/shared_ptr.hpp>
17#include <boost/lexical_cast.hpp>
18#include <boost/filesystem/path.hpp>
19#include <boost/filesystem/operations.hpp>
20
21#include <sqlite3.h>
22
23#include "app_state.hh"
24#include "cert.hh"
25#include "cleanup.hh"
26#include "constants.hh"
27#include "database.hh"
28#include "keys.hh"
29#include "sanity.hh"
30#include "schema_migration.hh"
31#include "transforms.hh"
32#include "ui.hh"
33#include "vocab.hh"
34#include "xdelta.hh"
35#include "epoch.hh"
36
37// defined in schema.sql, converted to header:
38#include "schema.h"
39
40// defined in views.sql, converted to header:
41#include "views.h"
42
43// this file defines a public, typed interface to the database.
44// the database class encapsulates all knowledge about sqlite,
45// the schema, and all SQL statements used to access the schema.
46//
47// see file schema.sql for the text of the schema.
48
49using boost::shared_ptr;
50using boost::lexical_cast;
51using namespace std;
52
53int const one_row = 1;
54int const one_col = 1;
55int const any_rows = -1;
56int const any_cols = -1;
57
58extern "C" {
59// some wrappers to ease migration
60 int sqlite3_exec_printf(sqlite3*,const char *sqlFormat,sqlite3_callback,
61 void *,char **errmsg,...);
62 int sqlite3_exec_vprintf(sqlite3*,const char *sqlFormat,sqlite3_callback,
63 void *,char **errmsg,va_list ap);
64 int sqlite3_get_table_vprintf(sqlite3*,const char *sqlFormat,char ***resultp,
65 int *nrow,int *ncolumn,char **errmsg,va_list ap);
66 const char *sqlite3_value_text_s(sqlite3_value *v);
67}
68
69int sqlite3_exec_printf(sqlite3 * db,
70 char const * sqlFormat,
71 sqlite3_callback cb,
72 void * user_data,
73 char ** errmsg,
74 ...)
75{
76 va_list ap;
77 va_start(ap, errmsg);
78 int result = sqlite3_exec_vprintf(db, sqlFormat, cb,
79 user_data, errmsg, ap);
80 va_end(ap);
81 return result;
82}
83
84int sqlite3_exec_vprintf(sqlite3 * db,
85 char const * sqlFormat,
86 sqlite3_callback cb,
87 void * user_data,
88 char ** errmsg,
89 va_list ap)
90{
91 char * formatted = sqlite3_vmprintf(sqlFormat, ap);
92 int result = sqlite3_exec(db, formatted, cb,
93 user_data, errmsg);
94 sqlite3_free(formatted);
95 return result;
96}
97
98int sqlite3_get_table_vprintf(sqlite3 * db,
99 char const * sqlFormat,
100 char *** resultp,
101 int * nrow,
102 int * ncolumn,
103 char ** errmsg,
104 va_list ap)
105{
106 char * formatted = sqlite3_vmprintf(sqlFormat, ap);
107 int result = sqlite3_get_table(db, formatted, resultp,
108 nrow, ncolumn, errmsg);
109 sqlite3_free(formatted);
110 return result;
111}
112
113database::database(fs::path const & fn) :
114 filename(fn),
115 // nb. update this if you change the schema. unfortunately we are not
116 // using self-digesting schemas due to comment irregularities and
117 // non-alphabetic ordering of tables in sql source files. we could create
118 // a temporary db, write our intended schema into it, and read it back,
119 // but this seems like it would be too rude. possibly revisit this issue.
120 schema("e372b508bea9b991816d1c74680f7ae10d2a6d94"),
121 __sql(NULL),
122 transaction_level(0)
123{}
124
125void
126database::check_schema()
127{
128 string db_schema_id;
129 calculate_schema_id (__sql, db_schema_id);
130 N (schema == db_schema_id,
131 F("database schemas do not match: "
132 "wanted %s, got %s. try migrating database")
133 % schema % db_schema_id);
134}
135
136// sqlite3_value_text gives a const unsigned char * but most of the time
137// we need a signed char
138const char *
139sqlite3_value_text_s(sqlite3_value *v)
140{
141 return (const char *)(sqlite3_value_text(v));
142}
143
144static void
145sqlite3_unbase64_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
146{
147 if (nargs != 1)
148 {
149 sqlite3_result_error(f, "need exactly 1 arg to unbase64()", -1);
150 return;
151 }
152 data decoded;
153 decode_base64(base64<data>(string(sqlite3_value_text_s(args[0]))), decoded);
154 sqlite3_result_blob(f, decoded().c_str(), decoded().size(), SQLITE_TRANSIENT);
155}
156
157static void
158sqlite3_unpack_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
159{
160 if (nargs != 1)
161 {
162 sqlite3_result_error(f, "need exactly 1 arg to unpack()", -1);
163 return;
164 }
165 data unpacked;
166 unpack(base64< gzip<data> >(string(sqlite3_value_text_s(args[0]))), unpacked);
167 sqlite3_result_blob(f, unpacked().c_str(), unpacked().size(), SQLITE_TRANSIENT);
168}
169
170void
171database::set_app(app_state * app)
172{
173 __app = app;
174}
175
176static void
177check_sqlite_format_version(fs::path const & filename)
178{
179 if (fs::exists(filename))
180 {
181 N(!fs::is_directory(filename),
182 F("database %s is a directory\n") % filename.string());
183
184 // sqlite 3 files begin with this constant string
185 // (version 2 files begin with a different one)
186 std::string version_string("SQLite format 3");
187
188 std::ifstream file(filename.string().c_str());
189 N(file, F("unable to probe database version in file %s") % filename.string());
190
191 for (std::string::const_iterator i = version_string.begin();
192 i != version_string.end(); ++i)
193 {
194 char c;
195 file.get(c);
196 N(c == *i, F("database %s is not an sqlite version 3 file, "
197 "try dump and reload") % filename.string());
198 }
199 }
200}
201
202
203struct sqlite3 *
204database::sql(bool init)
205{
206 if (! __sql)
207 {
208 N(!filename.empty(), F("no database specified"));
209
210 if (! init)
211 {
212 N(fs::exists(filename),
213 F("database %s does not exist") % filename.string());
214 N(!fs::is_directory(filename),
215 F("database %s is a directory") % filename.string());
216 }
217
218 check_sqlite_format_version(filename);
219 int error;
220 error = sqlite3_open(filename.string().c_str(), &__sql);
221 if (error)
222 throw oops(string("could not open database: ") + filename.string() +
223 (": " + string(sqlite3_errmsg(__sql))));
224 if (init)
225 execute(schema_constant);
226
227 check_schema();
228 install_functions(__app);
229 install_views();
230 }
231 return __sql;
232}
233
234void
235database::initialize()
236{
237 if (__sql)
238 throw oops("cannot initialize database while it is open");
239
240 N(!fs::exists(filename),
241 F("could not initialize database: %s: already exists")
242 % filename.string());
243
244 fs::path journal = mkpath(filename.string() + "-journal");
245 N(!fs::exists(journal),
246 F("existing (possibly stale) journal file '%s' "
247 "has same stem as new database '%s'")
248 % journal.string() % filename.string());
249
250 sqlite3 *s = sql(true);
251 I(s != NULL);
252}
253
254
255struct
256dump_request
257{
258 dump_request() {};
259 struct sqlite3 *sql;
260 string table_name;
261 ostream *out;
262};
263
264static int
265dump_row_cb(void *data, int n, char **vals, char **cols)
266{
267 dump_request *dump = reinterpret_cast<dump_request *>(data);
268 I(dump != NULL);
269 I(vals != NULL);
270 I(dump->out != NULL);
271 *(dump->out) << F("INSERT INTO %s VALUES(") % dump->table_name;
272 for (int i = 0; i < n; ++i)
273 {
274 if (i != 0)
275 *(dump->out) << ',';
276
277 if (vals[i] == NULL)
278 *(dump->out) << "NULL";
279 else
280 {
281 *(dump->out) << "'";
282 for (char *cp = vals[i]; *cp; ++cp)
283 {
284 if (*cp == '\'')
285 *(dump->out) << "''";
286 else
287 *(dump->out) << *cp;
288 }
289 *(dump->out) << "'";
290 }
291 }
292 *(dump->out) << ");\n";
293 return 0;
294}
295
296static int
297dump_table_cb(void *data, int n, char **vals, char **cols)
298{
299 dump_request *dump = reinterpret_cast<dump_request *>(data);
300 I(dump != NULL);
301 I(dump->sql != NULL);
302 I(vals != NULL);
303 I(vals[0] != NULL);
304 I(vals[1] != NULL);
305 I(vals[2] != NULL);
306 I(n == 3);
307 if (string(vals[1]) == "table")
308 {
309 *(dump->out) << vals[2] << ";\n";
310 dump->table_name = string(vals[0]);
311 sqlite3_exec_printf(dump->sql, "SELECT * FROM '%q'",
312 dump_row_cb, data, NULL, vals[0]);
313 }
314 return 0;
315}
316
317void
318database::dump(ostream & out)
319{
320 dump_request req;
321 req.out = &out;
322 req.sql = sql();
323 out << "BEGIN TRANSACTION;\n";
324 int res = sqlite3_exec(req.sql,
325 "SELECT name, type, sql FROM sqlite_master "
326 "WHERE type='table' AND sql NOT NULL "
327 "ORDER BY substr(type,2,1), name",
328 dump_table_cb, &req, NULL);
329 I(res == SQLITE_OK);
330 out << "COMMIT;\n";
331}
332
333void
334database::load(istream & in)
335{
336 char buf[constants::bufsz];
337 string tmp;
338
339 N(filename.string() != "",
340 F("need database name"));
341 N(!fs::exists(filename),
342 F("cannot create %s; it already exists\n") % filename.string());
343 int error = sqlite3_open(filename.string().c_str(), &__sql);
344 if (error)
345 throw oops(string("could not open database: ") + filename.string() +
346 (string(sqlite3_errmsg(__sql))));
347
348 while(in)
349 {
350 in.read(buf, constants::bufsz);
351 tmp.append(buf, in.gcount());
352 }
353
354 execute(tmp.c_str());
355}
356
357
358void
359database::debug(string const & sql, ostream & out)
360{
361 results res;
362 // "%s" construction prevents interpretation of %-signs in the query string
363 // as formatting commands.
364 fetch(res, any_cols, any_rows, "%s", sql.c_str());
365 out << "'" << sql << "' -> " << res.size() << " rows\n" << endl;
366 for (size_t i = 0; i < res.size(); ++i)
367 {
368 for (size_t j = 0; j < res[i].size(); ++j)
369 {
370 if (j != 0)
371 out << " | ";
372 out << res[i][j];
373 }
374 out << endl;
375 }
376}
377
378void
379database::info(ostream & out)
380{
381 string id;
382 calculate_schema_id(sql(), id);
383 unsigned long space = 0, tmp;
384 out << "schema version : " << id << endl;
385
386 out << "counts:" << endl;
387 out << " full manifests : " << count("manifests") << endl;
388 out << " manifest deltas : " << count("manifest_deltas") << endl;
389 out << " full files : " << count("files") << endl;
390 out << " file deltas : " << count("file_deltas") << endl;
391 out << " revisions : " << count("revisions") << endl;
392 out << " ancestry edges : " << count("revision_ancestry") << endl;
393 out << " certs : " << count("revision_certs") << endl;
394
395 out << "bytes:" << endl;
396 // FIXME: surely there is a less lame way to do this, that doesn't require
397 // updating every time the schema changes?
398 tmp = space_usage("manifests", "id || data");
399 space += tmp;
400 out << " full manifests : " << tmp << endl;
401
402 tmp = space_usage("manifest_deltas", "id || base || delta");
403 space += tmp;
404 out << " manifest deltas : " << tmp << endl;
405
406 tmp = space_usage("files", "id || data");
407 space += tmp;
408 out << " full files : " << tmp << endl;
409
410 tmp = space_usage("file_deltas", "id || base || delta");
411 space += tmp;
412 out << " file deltas : " << tmp << endl;
413
414 tmp = space_usage("revisions", "id || data");
415 space += tmp;
416 out << " revisions : " << tmp << endl;
417
418 tmp = space_usage("revision_ancestry", "parent || child");
419 space += tmp;
420 out << " cached ancestry : " << tmp << endl;
421
422 tmp = space_usage("revision_certs", "hash || id || name || value || keypair || signature");
423 space += tmp;
424 out << " certs : " << tmp << endl;
425
426 out << " total : " << space << endl;
427}
428
429void
430database::version(ostream & out)
431{
432 string id;
433 N(filename.string() != "",
434 F("need database name"));
435 int error = sqlite3_open(filename.string().c_str(), &__sql);
436 if (error)
437 {
438 sqlite3_close(__sql);
439 throw oops(string("could not open database: ") + filename.string() +
440 (": " + string(sqlite3_errmsg(__sql))));
441 }
442 calculate_schema_id(__sql, id);
443 sqlite3_close(__sql);
444 out << "database schema version: " << id << endl;
445}
446
447void
448database::migrate()
449{
450 N(filename.string() != "",
451 F("need database name"));
452 int error = sqlite3_open(filename.string().c_str(), &__sql);
453 if (error)
454 {
455 sqlite3_close(__sql);
456 throw oops(string("could not open database: ") + filename.string() +
457 (": " + string(sqlite3_errmsg(__sql))));
458 }
459 migrate_monotone_schema(__sql);
460 sqlite3_close(__sql);
461}
462
463void
464database::rehash()
465{
466 transaction_guard guard(*this);
467 ticker mcerts("mcerts", "m", 1);
468 ticker pubkeys("pubkeys", "+", 1);
469 ticker privkeys("privkeys", "!", 1);
470
471 {
472 // rehash all mcerts
473 results res;
474 vector<cert> certs;
475 fetch(res, 5, any_rows,
476 "SELECT id, name, value, keypair, signature "
477 "FROM manifest_certs");
478 results_to_certs(res, certs);
479 execute("DELETE FROM manifest_certs");
480 for(vector<cert>::const_iterator i = certs.begin(); i != certs.end(); ++i)
481 {
482 put_cert(*i, "manifest_certs");
483 ++mcerts;
484 }
485 }
486
487 {
488 // rehash all pubkeys
489 results res;
490 fetch(res, 2, any_rows, "SELECT id, keydata FROM public_keys");
491 execute("DELETE FROM public_keys");
492 for (size_t i = 0; i < res.size(); ++i)
493 {
494 hexenc<id> tmp;
495 key_hash_code(rsa_keypair_id(res[i][0]), base64<rsa_pub_key>(res[i][1]), tmp);
496 execute("INSERT INTO public_keys VALUES('%q', '%q', '%q')",
497 tmp().c_str(), res[i][0].c_str(), res[i][1].c_str());
498 ++pubkeys;
499 }
500 }
501
502 {
503 // rehash all privkeys
504 results res;
505 fetch(res, 2, any_rows, "SELECT id, keydata FROM private_keys");
506 execute("DELETE FROM private_keys");
507 for (size_t i = 0; i < res.size(); ++i)
508 {
509 hexenc<id> tmp;
510 key_hash_code(rsa_keypair_id(res[i][0]), base64< arc4<rsa_priv_key> >(res[i][1]), tmp);
511 execute("INSERT INTO private_keys VALUES('%q', '%q', '%q')",
512 tmp().c_str(), res[i][0].c_str(), res[i][1].c_str());
513 ++privkeys;
514 }
515 }
516
517 guard.commit();
518}
519
520void
521database::ensure_open()
522{
523 sqlite3 *s = sql();
524 I(s != NULL);
525}
526
527database::~database()
528{
529 if (__sql)
530 {
531 sqlite3_close(__sql);
532 __sql = 0;
533 }
534}
535
536static void
537assert_sqlite3_ok(int res)
538{
539 switch (res)
540 {
541 case SQLITE_OK:
542 break;
543
544 case SQLITE_ERROR:
545 throw oops("SQL error or missing database");
546 break;
547
548 case SQLITE_INTERNAL:
549 throw oops("An internal logic error in SQLite");
550 break;
551
552 case SQLITE_PERM:
553 throw oops("Access permission denied");
554 break;
555
556 case SQLITE_ABORT:
557 throw oops("Callback routine requested an abort");
558 break;
559
560 case SQLITE_BUSY:
561 throw oops("The database file is locked");
562 break;
563
564 case SQLITE_LOCKED:
565 throw oops("A table in the database is locked");
566 break;
567
568 case SQLITE_NOMEM:
569 throw oops("A malloc() failed");
570 break;
571
572 case SQLITE_READONLY:
573 throw oops("Attempt to write a readonly database");
574 break;
575
576 case SQLITE_INTERRUPT:
577 throw oops("Operation terminated by sqlite3_interrupt()");
578 break;
579
580 case SQLITE_IOERR:
581 throw oops("Some kind of disk I/O error occurred");
582 break;
583
584 case SQLITE_CORRUPT:
585 throw oops("The database disk image is malformed");
586 break;
587
588 case SQLITE_NOTFOUND:
589 throw oops("(Internal Only) Table or record not found");
590 break;
591
592 case SQLITE_FULL:
593 throw oops("Insertion failed because database (or filesystem) is full");
594 break;
595
596 case SQLITE_CANTOPEN:
597 throw oops("Unable to open the database file");
598 break;
599
600 case SQLITE_PROTOCOL:
601 throw oops("database lock protocol error");
602 break;
603
604 case SQLITE_EMPTY:
605 throw oops("(Internal Only) database table is empty");
606 break;
607
608 case SQLITE_SCHEMA:
609 throw oops("The database schema changed");
610 break;
611
612 case SQLITE_TOOBIG:
613 throw oops("Too much data for one row of a table");
614 break;
615
616 case SQLITE_CONSTRAINT:
617 throw oops("Abort due to contraint violation");
618 break;
619
620 case SQLITE_MISMATCH:
621 throw oops("Data type mismatch");
622 break;
623
624 case SQLITE_MISUSE:
625 throw oops("Library used incorrectly");
626 break;
627
628 case SQLITE_NOLFS:
629 throw oops("Uses OS features not supported on host");
630 break;
631
632 case SQLITE_AUTH:
633 throw oops("Authorization denied");
634 break;
635
636 default:
637 throw oops(string("Unknown DB result code: ") + lexical_cast<string>(res));
638 break;
639 }
640}
641
642void
643database::execute(char const * query, ...)
644{
645 va_list ap;
646 int res;
647 char * errmsg = NULL;
648
649 va_start(ap, query);
650
651 // log it
652 char * formatted = sqlite3_vmprintf(query, ap);
653 string qq(formatted);
654 if (qq.size() > constants::db_log_line_sz)
655 qq = qq.substr(0, constants::db_log_line_sz) + string(" ...");
656 L(F("db.execute(\"%s\")\n") % qq);
657 sqlite3_free(formatted);
658
659 va_end(ap);
660 va_start(ap, query);
661
662 // do it
663 res = sqlite3_exec_vprintf(sql(), query, NULL, NULL, &errmsg, ap);
664
665 va_end(ap);
666
667 if (errmsg)
668 throw oops(string("sqlite exec error ") + errmsg);
669
670 assert_sqlite3_ok(res);
671
672}
673
674void
675database::fetch(results & res,
676 int const want_cols,
677 int const want_rows,
678 char const * query, ...)
679{
680 char ** result = NULL;
681 int nrow;
682 int ncol;
683 char * errmsg = NULL;
684 int rescode;
685
686 va_list ap;
687 res.clear();
688 res.resize(0);
689 va_start(ap, query);
690
691 // log it
692 char * formatted = sqlite3_vmprintf(query, ap);
693 string qq(formatted);
694 if (qq.size() > constants::log_line_sz)
695 qq = qq.substr(0, constants::log_line_sz) + string(" ...");
696 L(F("db.fetch(\"%s\")\n") % qq);
697 sqlite3_free(formatted);
698
699 va_end(ap);
700 va_start(ap, query);
701
702 // do it
703 rescode = sqlite3_get_table_vprintf(sql(), query, &result, &nrow, &ncol, &errmsg, ap);
704
705 va_end(ap);
706
707 cleanup_ptr<char **, void>
708 result_guard(result, &sqlite3_free_table);
709
710 string ctx = string("db query [") + string(query) + "]: ";
711
712 if (errmsg)
713 throw oops(ctx + string("sqlite error ") + errmsg);
714 assert_sqlite3_ok(rescode);
715
716 if (want_cols == 0 && ncol == 0) return;
717 if (want_rows == 0 && nrow == 0) return;
718 if (want_cols == any_rows && ncol == 0) return;
719 if (want_rows == any_rows && nrow == 0) return;
720
721 if (want_cols != any_cols &&
722 ncol != want_cols)
723 throw oops((F("%s wanted %d columns, got %s")
724 % ctx % want_cols % ncol).str());
725
726 if (want_rows != any_rows &&
727 nrow != want_rows)
728 throw oops((F("%s wanted %d rows, got %s")
729 % ctx % want_rows % nrow).str());
730
731 if (!result)
732 throw oops(ctx + "null result set");
733
734 for (int i = 0; i < ncol; ++i)
735 if (!result[i])
736 throw oops(ctx + "null column name");
737
738 for (int row = 0; row < nrow; ++row)
739 {
740 vector<string> rowvec;
741 for (int col = 0; col < ncol; ++col)
742 {
743 int i = ((1 + row) * ncol) + col;
744 if (!result[i])
745 throw oops(ctx + "null result value");
746 else
747 rowvec.push_back(result[i]);
748 }
749 res.push_back(rowvec);
750 }
751}
752
753// general application-level logic
754
755void
756database::set_filename(fs::path const & file)
757{
758 if (__sql)
759 {
760 throw oops("cannot change filename to " + file.string() + " while db is open");
761 }
762 filename = file;
763}
764
765void
766database::begin_transaction()
767{
768 if (transaction_level == 0)
769 execute("BEGIN");
770 transaction_level++;
771}
772
773void
774database::commit_transaction()
775{
776 if (transaction_level == 1)
777 execute("COMMIT");
778 transaction_level--;
779}
780
781void
782database::rollback_transaction()
783{
784 if (transaction_level == 1)
785 execute("ROLLBACK");
786 transaction_level--;
787}
788
789
790bool
791database::exists(hexenc<id> const & ident,
792 string const & table)
793{
794 results res;
795 fetch(res, one_col, any_rows,
796 "SELECT id FROM '%q' WHERE id = '%q'",
797 table.c_str(), ident().c_str());
798 I((res.size() == 1) || (res.size() == 0));
799 return res.size() == 1;
800}
801
802
803bool
804database::delta_exists(hexenc<id> const & ident,
805 string const & table)
806{
807 results res;
808 fetch(res, one_col, any_rows,
809 "SELECT id FROM '%q' WHERE id = '%q'",
810 table.c_str(), ident().c_str());
811 return res.size() > 0;
812}
813
814bool
815database::delta_exists(hexenc<id> const & ident,
816 hexenc<id> const & base,
817 string const & table)
818{
819 results res;
820 fetch(res, one_col, any_rows,
821 "SELECT id FROM '%q' WHERE id = '%q' AND base = '%q'",
822 table.c_str(), ident().c_str(), base().c_str());
823 I((res.size() == 1) || (res.size() == 0));
824 return res.size() == 1;
825}
826
827unsigned long
828database::count(string const & table)
829{
830 results res;
831 fetch(res, one_col, one_row,
832 "SELECT COUNT(*) FROM '%q'",
833 table.c_str());
834 return lexical_cast<unsigned long>(res[0][0]);
835}
836
837unsigned long
838database::space_usage(string const & table, string const & concatenated_columns)
839{
840 results res;
841 fetch(res, one_col, one_row,
842 "SELECT SUM(LENGTH(%s)) FROM '%q'",
843 concatenated_columns.c_str(), table.c_str());
844 return lexical_cast<unsigned long>(res[0][0]);
845}
846
847void
848database::get_ids(string const & table, set< hexenc<id> > & ids)
849{
850 results res;
851
852 fetch(res, one_col, any_rows, "SELECT id FROM %q", table.c_str());
853
854 for (size_t i = 0; i < res.size(); ++i)
855 {
856 ids.insert(hexenc<id>(res[i][0]));
857 }
858}
859
860void
861database::get(hexenc<id> const & ident,
862 data & dat,
863 string const & table)
864{
865 results res;
866 fetch(res, one_col, one_row,
867 "SELECT data FROM '%q' WHERE id = '%q'",
868 table.c_str(), ident().c_str());
869
870 // consistency check
871 base64<gzip<data> > rdata(res[0][0]);
872 data rdata_unpacked;
873 unpack(rdata, rdata_unpacked);
874
875 hexenc<id> tid;
876 calculate_ident(rdata_unpacked, tid);
877 I(tid == ident);
878
879 dat = rdata_unpacked;
880}
881
882void
883database::get_delta(hexenc<id> const & ident,
884 hexenc<id> const & base,
885 delta & del,
886 string const & table)
887{
888 I(ident() != "");
889 I(base() != "");
890 results res;
891 fetch(res, one_col, one_row,
892 "SELECT delta FROM '%q' WHERE id = '%q' AND base = '%q'",
893 table.c_str(), ident().c_str(), base().c_str());
894
895 base64<gzip<delta> > del_packed = res[0][0];
896 unpack(del_packed, del);
897}
898
899void
900database::put(hexenc<id> const & ident,
901 data const & dat,
902 string const & table)
903{
904 // consistency check
905 I(ident() != "");
906 hexenc<id> tid;
907 calculate_ident(dat, tid);
908 I(tid == ident);
909
910 base64<gzip<data> > dat_packed;
911 pack(dat, dat_packed);
912
913 execute("INSERT INTO '%q' VALUES('%q', '%q')",
914 table.c_str(), ident().c_str(), dat_packed().c_str());
915}
916void
917database::put_delta(hexenc<id> const & ident,
918 hexenc<id> const & base,
919 delta const & del,
920 string const & table)
921{
922 // nb: delta schema is (id, base, delta)
923 I(ident() != "");
924 I(base() != "");
925
926 base64<gzip<delta> > del_packed;
927 pack(del, del_packed);
928
929 execute("INSERT INTO '%q' VALUES('%q', '%q', '%q')",
930 table.c_str(),
931 ident().c_str(), base().c_str(), del_packed().c_str());
932}
933
934// static ticker cache_hits("vcache hits", "h", 1);
935
936struct version_cache
937{
938 size_t capacity;
939 size_t use;
940 std::map<hexenc<id>, data> cache;
941
942 version_cache() : capacity(constants::db_version_cache_sz), use(0)
943 {
944 srand(time(NULL));
945 }
946
947 void put(hexenc<id> const & ident, data const & dat)
948 {
949 while (!cache.empty()
950 && use + dat().size() > capacity)
951 {
952 std::string key = (F("%08.8x%08.8x%08.8x%08.8x%08.8x")
953 % rand() % rand() % rand() % rand() % rand()).str();
954 std::map<hexenc<id>, data>::const_iterator i;
955 i = cache.lower_bound(hexenc<id>(key));
956 if (i == cache.end())
957 {
958 // we can't find a random entry, probably there's only one
959 // entry and we missed it. delete first entry instead.
960 i = cache.begin();
961 }
962 I(i != cache.end());
963 I(use >= i->second().size());
964 L(F("version cache expiring %s\n") % i->first);
965 use -= i->second().size();
966 cache.erase(i->first);
967 }
968 cache.insert(std::make_pair(ident, dat));
969 use += dat().size();
970 }
971
972 bool exists(hexenc<id> const & ident)
973 {
974 std::map<hexenc<id>, data>::const_iterator i;
975 i = cache.find(ident);
976 return i != cache.end();
977 }
978
979 bool get(hexenc<id> const & ident, data & dat)
980 {
981 std::map<hexenc<id>, data>::const_iterator i;
982 i = cache.find(ident);
983 if (i == cache.end())
984 return false;
985 // ++cache_hits;
986 L(F("version cache hit on %s\n") % ident);
987 dat = i->second;
988 return true;
989 }
990};
991
992static version_cache vcache;
993
994void
995database::get_version(hexenc<id> const & ident,
996 data & dat,
997 string const & data_table,
998 string const & delta_table)
999{
1000 I(ident() != "");
1001
1002 if (vcache.get(ident, dat))
1003 {
1004 return;
1005 }
1006 else if (exists(ident, data_table))
1007 {
1008 // easy path
1009 get(ident, dat, data_table);
1010 }
1011 else
1012 {
1013 // tricky path
1014
1015 // we start from the file we want to reconstruct and work *forwards*
1016 // through the database, until we get to a full data object. we then
1017 // trace back through the list of edges we followed to get to the data
1018 // object, applying reverse deltas.
1019 //
1020 // the effect of this algorithm is breadth-first search, backwards
1021 // through the storage graph, to discover a forwards shortest path, and
1022 // then following that shortest path with delta application.
1023 //
1024 // we used to do this with the boost graph library, but it invovled
1025 // loading too much of the storage graph into memory at any moment. this
1026 // imperative version only loads the descendents of the reconstruction
1027 // node, so it much cheaper in terms of memory.
1028 //
1029 // we also maintain a cycle-detecting set, just to be safe
1030
1031 L(F("reconstructing %s in %s\n") % ident % delta_table);
1032 I(delta_exists(ident, delta_table));
1033
1034 // nb: an edge map goes in the direction of the
1035 // delta, *not* the direction we discover things in,
1036 // i.e. each map is of the form [newid] -> [oldid]
1037
1038 typedef map< hexenc<id>, hexenc<id> > edgemap;
1039 list< shared_ptr<edgemap> > paths;
1040
1041 set< hexenc<id> > frontier, cycles;
1042 frontier.insert(ident);
1043
1044 bool found_root = false;
1045 hexenc<id> root("");
1046
1047 while (! found_root)
1048 {
1049 set< hexenc<id> > next_frontier;
1050 shared_ptr<edgemap> frontier_map(new edgemap());
1051
1052 I(!frontier.empty());
1053
1054 for (set< hexenc<id> >::const_iterator i = frontier.begin();
1055 i != frontier.end(); ++i)
1056 {
1057 if (vcache.exists(*i) || exists(*i, data_table))
1058 {
1059 root = *i;
1060 found_root = true;
1061 break;
1062 }
1063 else
1064 {
1065 cycles.insert(*i);
1066 results res;
1067 fetch(res, one_col, any_rows, "SELECT base from '%q' WHERE id = '%q'",
1068 delta_table.c_str(), (*i)().c_str());
1069 for (size_t k = 0; k < res.size(); ++k)
1070 {
1071 hexenc<id> const nxt(res[k][0]);
1072
1073 if (cycles.find(nxt) != cycles.end())
1074 throw oops("cycle in table '" + delta_table + "', at node "
1075 + (*i)() + " <- " + nxt());
1076
1077 next_frontier.insert(nxt);
1078
1079 if (frontier_map->find(nxt) == frontier_map->end())
1080 {
1081 L(F("inserting edge: %s <- %s\n") % (*i) % nxt);
1082 frontier_map->insert(make_pair(nxt, *i));
1083 }
1084 else
1085 L(F("skipping merge edge %s <- %s\n") % (*i) % nxt);
1086 }
1087 }
1088 }
1089 if (!found_root)
1090 {
1091 frontier = next_frontier;
1092 paths.push_front(frontier_map);
1093 }
1094 }
1095
1096 // path built, now all we need to do is follow it back
1097
1098 I(found_root);
1099 I(root() != "");
1100 data begin;
1101
1102 if (vcache.exists(root))
1103 {
1104 I(vcache.get(root, begin));
1105 }
1106 else
1107 {
1108 get(root, begin, data_table);
1109 }
1110
1111 hexenc<id> curr = root;
1112
1113 boost::shared_ptr<delta_applicator> app = new_piecewise_applicator();
1114 app->begin(begin());
1115
1116 for (list< shared_ptr<edgemap> >::const_iterator p = paths.begin();
1117 p != paths.end(); ++p)
1118 {
1119 shared_ptr<edgemap> i = *p;
1120 I(i->find(curr) != i->end());
1121 hexenc<id> const nxt = i->find(curr)->second;
1122
1123 if (!vcache.exists(curr))
1124 {
1125 string tmp;
1126 app->finish(tmp);
1127 vcache.put(curr, tmp);
1128 }
1129
1130
1131 L(F("following delta %s -> %s\n") % curr % nxt);
1132 delta del;
1133 get_delta(nxt, curr, del, delta_table);
1134 apply_delta (app, del());
1135
1136 app->next();
1137 curr = nxt;
1138 }
1139
1140 string tmp;
1141 app->finish(tmp);
1142 dat = data(tmp);
1143
1144 hexenc<id> final;
1145 calculate_ident(dat, final);
1146 I(final == ident);
1147 }
1148 vcache.put(ident, dat);
1149}
1150
1151
1152void
1153database::drop(hexenc<id> const & ident,
1154 string const & table)
1155{
1156 execute("DELETE FROM '%q' WHERE id = '%q'",
1157 table.c_str(),
1158 ident().c_str());
1159}
1160
1161void
1162database::put_version(hexenc<id> const & old_id,
1163 hexenc<id> const & new_id,
1164 delta const & del,
1165 string const & data_table,
1166 string const & delta_table)
1167{
1168
1169 data old_data, new_data;
1170 delta reverse_delta;
1171
1172 get_version(old_id, old_data, data_table, delta_table);
1173 patch(old_data, del, new_data);
1174 diff(new_data, old_data, reverse_delta);
1175
1176 transaction_guard guard(*this);
1177 if (exists(old_id, data_table))
1178 {
1179 // descendent of a head version replaces the head, therefore old head
1180 // must be disposed of
1181 drop(old_id, data_table);
1182 }
1183 put(new_id, new_data, data_table);
1184 put_delta(old_id, new_id, reverse_delta, delta_table);
1185 guard.commit();
1186}
1187
1188void
1189database::put_reverse_version(hexenc<id> const & new_id,
1190 hexenc<id> const & old_id,
1191 delta const & reverse_del,
1192 string const & data_table,
1193 string const & delta_table)
1194{
1195 data old_data, new_data;
1196
1197 get_version(new_id, new_data, data_table, delta_table);
1198 patch(new_data, reverse_del, old_data);
1199 hexenc<id> check;
1200 calculate_ident(old_data, check);
1201 I(old_id == check);
1202
1203 transaction_guard guard(*this);
1204 put_delta(old_id, new_id, reverse_del, delta_table);
1205 guard.commit();
1206}
1207
1208
1209
1210// ------------------------------------------------------------
1211// -- --
1212// -- public interface follows --
1213// -- --
1214// ------------------------------------------------------------
1215
1216bool
1217database::file_version_exists(file_id const & id)
1218{
1219 return delta_exists(id.inner(), "file_deltas")
1220 || exists(id.inner(), "files");
1221}
1222
1223bool
1224database::manifest_version_exists(manifest_id const & id)
1225{
1226 return delta_exists(id.inner(), "manifest_deltas")
1227 || exists(id.inner(), "manifests");
1228}
1229
1230bool
1231database::revision_exists(revision_id const & id)
1232{
1233 return exists(id.inner(), "revisions");
1234}
1235
1236void
1237database::get_file_ids(set<file_id> & ids)
1238{
1239 ids.clear();
1240 set< hexenc<id> > tmp;
1241 get_ids("files", tmp);
1242 get_ids("file_deltas", tmp);
1243 ids.insert(tmp.begin(), tmp.end());
1244}
1245
1246void
1247database::get_manifest_ids(set<manifest_id> & ids)
1248{
1249 ids.clear();
1250 set< hexenc<id> > tmp;
1251 get_ids("manifests", tmp);
1252 get_ids("manifest_deltas", tmp);
1253 ids.insert(tmp.begin(), tmp.end());
1254}
1255
1256void
1257database::get_revision_ids(set<revision_id> & ids)
1258{
1259 ids.clear();
1260 set< hexenc<id> > tmp;
1261 get_ids("revisions", tmp);
1262 ids.insert(tmp.begin(), tmp.end());
1263}
1264
1265void
1266database::get_file_version(file_id const & id,
1267 file_data & dat)
1268{
1269 data tmp;
1270 get_version(id.inner(), tmp, "files", "file_deltas");
1271 dat = tmp;
1272}
1273
1274void
1275database::get_manifest_version(manifest_id const & id,
1276 manifest_data & dat)
1277{
1278 data tmp;
1279 get_version(id.inner(), tmp, "manifests", "manifest_deltas");
1280 dat = tmp;
1281}
1282
1283void
1284database::get_manifest(manifest_id const & id,
1285 manifest_map & mm)
1286{
1287 manifest_data mdat;
1288 get_manifest_version(id, mdat);
1289 read_manifest_map(mdat, mm);
1290}
1291
1292
1293void
1294database::put_file(file_id const & id,
1295 file_data const & dat)
1296{
1297 put(id.inner(), dat.inner(), "files");
1298}
1299
1300void
1301database::put_file_version(file_id const & old_id,
1302 file_id const & new_id,
1303 file_delta const & del)
1304{
1305 put_version(old_id.inner(), new_id.inner(), del.inner(),
1306 "files", "file_deltas");
1307}
1308
1309void
1310database::put_file_reverse_version(file_id const & new_id,
1311 file_id const & old_id,
1312 file_delta const & del)
1313{
1314 put_reverse_version(new_id.inner(), old_id.inner(), del.inner(),
1315 "files", "file_deltas");
1316}
1317
1318
1319void
1320database::put_manifest(manifest_id const & id,
1321 manifest_data const & dat)
1322{
1323 put(id.inner(), dat.inner(), "manifests");
1324}
1325
1326void
1327database::put_manifest_version(manifest_id const & old_id,
1328 manifest_id const & new_id,
1329 manifest_delta const & del)
1330{
1331 put_version(old_id.inner(), new_id.inner(), del.inner(),
1332 "manifests", "manifest_deltas");
1333}
1334
1335void
1336database::put_manifest_reverse_version(manifest_id const & new_id,
1337 manifest_id const & old_id,
1338 manifest_delta const & del)
1339{
1340 put_reverse_version(new_id.inner(), old_id.inner(), del.inner(),
1341 "manifests", "manifest_deltas");
1342}
1343
1344
1345void
1346database::get_revision_ancestry(std::multimap<revision_id, revision_id> & graph)
1347{
1348 results res;
1349 graph.clear();
1350 fetch(res, 2, any_rows,
1351 "SELECT parent,child FROM revision_ancestry");
1352 for (size_t i = 0; i < res.size(); ++i)
1353 graph.insert(std::make_pair(revision_id(res[i][0]),
1354 revision_id(res[i][1])));
1355}
1356
1357void
1358database::get_revision_parents(revision_id const & id,
1359 set<revision_id> & parents)
1360{
1361 I(!null_id(id));
1362 results res;
1363 parents.clear();
1364 fetch(res, one_col, any_rows,
1365 "SELECT parent FROM revision_ancestry WHERE child = '%q'",
1366 id.inner()().c_str());
1367 for (size_t i = 0; i < res.size(); ++i)
1368 parents.insert(revision_id(res[i][0]));
1369}
1370
1371void
1372database::get_revision_children(revision_id const & id,
1373 set<revision_id> & children)
1374{
1375 I(!null_id(id));
1376 results res;
1377 children.clear();
1378 fetch(res, one_col, any_rows,
1379 "SELECT child FROM revision_ancestry WHERE parent = '%q'",
1380 id.inner()().c_str());
1381 for (size_t i = 0; i < res.size(); ++i)
1382 children.insert(revision_id(res[i][0]));
1383}
1384
1385void
1386database::get_revision_manifest(revision_id const & rid,
1387 manifest_id & mid)
1388{
1389 revision_set rev;
1390 get_revision(rid, rev);
1391 mid = rev.new_manifest;
1392}
1393
1394void
1395database::get_revision(revision_id const & id,
1396 revision_set & rev)
1397{
1398 revision_data d;
1399 get_revision(id, d);
1400 read_revision_set(d, rev);
1401}
1402
1403void
1404database::get_revision(revision_id const & id,
1405 revision_data & dat)
1406{
1407 I(!null_id(id));
1408 results res;
1409 fetch(res, one_col, one_row,
1410 "SELECT data FROM revisions WHERE id = '%q'",
1411 id.inner()().c_str());
1412
1413 base64<gzip<data> > rdat_packed;
1414 rdat_packed = base64<gzip<data> >(res[0][0]);
1415 data rdat;
1416 unpack(rdat_packed, rdat);
1417
1418 // verify that we got a revision with the right id
1419 {
1420 revision_id tmp;
1421 calculate_ident(rdat, tmp);
1422 I(id == tmp);
1423 }
1424
1425 dat = rdat;
1426}
1427
1428void
1429database::put_revision(revision_id const & new_id,
1430 revision_set const & rev)
1431{
1432
1433 I(!null_id(new_id));
1434 I(!revision_exists(new_id));
1435 revision_data d;
1436
1437 rev.check_sane();
1438
1439 write_revision_set(rev, d);
1440 revision_id tmp;
1441 calculate_ident(d, tmp);
1442 I(tmp == new_id);
1443
1444 base64<gzip<data> > d_packed;
1445 pack(d.inner(), d_packed);
1446
1447 transaction_guard guard(*this);
1448
1449 execute("INSERT INTO revisions VALUES('%q', '%q')",
1450 new_id.inner()().c_str(),
1451 d_packed().c_str());
1452
1453 for (edge_map::const_iterator e = rev.edges.begin();
1454 e != rev.edges.end(); ++e)
1455 {
1456 execute("INSERT INTO revision_ancestry VALUES('%q', '%q')",
1457 edge_old_revision(e).inner()().c_str(),
1458 new_id.inner()().c_str());
1459 }
1460
1461 check_sane_history(new_id, constants::verify_depth, *__app);
1462
1463 guard.commit();
1464}
1465
1466void
1467database::put_revision(revision_id const & new_id,
1468 revision_data const & dat)
1469{
1470 revision_set rev;
1471 read_revision_set(dat, rev);
1472 put_revision(new_id, rev);
1473}
1474
1475
1476void
1477database::delete_existing_revs_and_certs()
1478{
1479 execute("DELETE from revisions");
1480 execute("DELETE from revision_ancestry");
1481 execute("DELETE from revision_certs");
1482}
1483
1484/// Deletes one revision from the local database.
1485/// @see kill_rev_locally
1486void
1487database::delete_existing_rev_and_certs(revision_id const & rid){
1488
1489 //check that the revision exists and doesn't have any children
1490 I(revision_exists(rid));
1491 set<revision_id> children;
1492 get_revision_children(rid, children);
1493 I(!children.size());
1494
1495 // perform the actual SQL transactions to kill rev rid here
1496 L(F("Killing revision %s locally\n") % rid);
1497 execute("DELETE from revision_certs WHERE id = '%s'",rid.inner()().c_str());
1498 execute("DELETE from revision_ancestry WHERE child = '%s'",
1499 rid.inner()().c_str());
1500 execute("DELETE from revisions WHERE id = '%s'",rid.inner()().c_str());
1501}
1502
1503// crypto key management
1504
1505void
1506database::get_key_ids(string const & pattern,
1507 vector<rsa_keypair_id> & pubkeys,
1508 vector<rsa_keypair_id> & privkeys)
1509{
1510 pubkeys.clear();
1511 privkeys.clear();
1512 results res;
1513
1514 if (pattern != "")
1515 fetch(res, one_col, any_rows,
1516 "SELECT id from public_keys WHERE id GLOB '%q'",
1517 pattern.c_str());
1518 else
1519 fetch(res, one_col, any_rows,
1520 "SELECT id from public_keys");
1521
1522 for (size_t i = 0; i < res.size(); ++i)
1523 pubkeys.push_back(res[i][0]);
1524
1525 if (pattern != "")
1526 fetch(res, one_col, any_rows,
1527 "SELECT id from private_keys WHERE id GLOB '%q'",
1528 pattern.c_str());
1529 else
1530 fetch(res, one_col, any_rows,
1531 "SELECT id from private_keys");
1532
1533 for (size_t i = 0; i < res.size(); ++i)
1534 privkeys.push_back(res[i][0]);
1535}
1536
1537void
1538database::get_keys(string const & table, vector<rsa_keypair_id> & keys)
1539{
1540 keys.clear();
1541 results res;
1542 fetch(res, one_col, any_rows, "SELECT id from '%q'", table.c_str());
1543 for (size_t i = 0; i < res.size(); ++i)
1544 keys.push_back(res[i][0]);
1545}
1546
1547void
1548database::get_private_keys(vector<rsa_keypair_id> & keys)
1549{
1550 get_keys("private_keys", keys);
1551}
1552
1553void
1554database::get_public_keys(vector<rsa_keypair_id> & keys)
1555{
1556 get_keys("public_keys", keys);
1557}
1558
1559bool
1560database::public_key_exists(hexenc<id> const & hash)
1561{
1562 results res;
1563 fetch(res, one_col, any_rows,
1564 "SELECT id FROM public_keys WHERE hash = '%q'",
1565 hash().c_str());
1566 I((res.size() == 1) || (res.size() == 0));
1567 if (res.size() == 1)
1568 return true;
1569 return false;
1570}
1571
1572bool
1573database::public_key_exists(rsa_keypair_id const & id)
1574{
1575 results res;
1576 fetch(res, one_col, any_rows,
1577 "SELECT id FROM public_keys WHERE id = '%q'",
1578 id().c_str());
1579 I((res.size() == 1) || (res.size() == 0));
1580 if (res.size() == 1)
1581 return true;
1582 return false;
1583}
1584
1585bool
1586database::private_key_exists(rsa_keypair_id const & id)
1587{
1588 results res;
1589 fetch(res, one_col, any_rows,
1590 "SELECT id FROM private_keys WHERE id = '%q'",
1591 id().c_str());
1592 I((res.size() == 1) || (res.size() == 0));
1593 if (res.size() == 1)
1594 return true;
1595 return false;
1596}
1597
1598bool
1599database::key_exists(rsa_keypair_id const & id)
1600{
1601 return public_key_exists(id) || private_key_exists(id);
1602}
1603
1604void
1605database::get_pubkey(hexenc<id> const & hash,
1606 rsa_keypair_id & id,
1607 base64<rsa_pub_key> & pub_encoded)
1608{
1609 results res;
1610 fetch(res, 2, one_row,
1611 "SELECT id, keydata FROM public_keys where hash = '%q'",
1612 hash().c_str());
1613 id = res[0][0];
1614 pub_encoded = res[0][1];
1615}
1616
1617void
1618database::get_key(rsa_keypair_id const & pub_id,
1619 base64<rsa_pub_key> & pub_encoded)
1620{
1621 results res;
1622 fetch(res, one_col, one_row,
1623 "SELECT keydata FROM public_keys where id = '%q'",
1624 pub_id().c_str());
1625 pub_encoded = res[0][0];
1626}
1627
1628void
1629database::get_key(rsa_keypair_id const & priv_id,
1630 base64< arc4<rsa_priv_key> > & priv_encoded)
1631{
1632 results res;
1633 fetch(res, one_col, one_col,
1634 "SELECT keydata FROM private_keys where id = '%q'",
1635 priv_id().c_str());
1636 priv_encoded = res[0][0];
1637}
1638
1639void
1640database::put_key(rsa_keypair_id const & pub_id,
1641 base64<rsa_pub_key> const & pub_encoded)
1642{
1643 hexenc<id> thash;
1644 key_hash_code(pub_id, pub_encoded, thash);
1645 I(!public_key_exists(thash));
1646 E(!public_key_exists(pub_id),
1647 F("another key with name '%s' already exists") % pub_id);
1648 execute("INSERT INTO public_keys VALUES('%q', '%q', '%q')",
1649 thash().c_str(), pub_id().c_str(), pub_encoded().c_str());
1650}
1651
1652void
1653database::put_key(rsa_keypair_id const & priv_id,
1654 base64< arc4<rsa_priv_key> > const & priv_encoded)
1655{
1656 hexenc<id> thash;
1657 key_hash_code(priv_id, priv_encoded, thash);
1658 E(!private_key_exists(priv_id),
1659 F("another key with name '%s' already exists") % priv_id);
1660 execute("INSERT INTO private_keys VALUES('%q', '%q', '%q')",
1661 thash().c_str(), priv_id().c_str(), priv_encoded().c_str());
1662}
1663
1664void
1665database::put_key_pair(rsa_keypair_id const & id,
1666 base64<rsa_pub_key> const & pub_encoded,
1667 base64< arc4<rsa_priv_key> > const & priv_encoded)
1668{
1669 transaction_guard guard(*this);
1670 put_key(id, pub_encoded);
1671 put_key(id, priv_encoded);
1672 guard.commit();
1673}
1674
1675void
1676database::delete_private_key(rsa_keypair_id const & pub_id)
1677{
1678 execute("DELETE FROM private_keys WHERE id = '%q'",
1679 pub_id().c_str());
1680}
1681
1682void
1683database::delete_public_key(rsa_keypair_id const & pub_id)
1684{
1685 execute("DELETE FROM public_keys WHERE id = '%q'",
1686 pub_id().c_str());
1687}
1688
1689// cert management
1690
1691bool
1692database::cert_exists(cert const & t,
1693 string const & table)
1694{
1695 results res;
1696 fetch(res, 1, any_rows,
1697 "SELECT id FROM '%q' WHERE id = '%q' "
1698 "AND name = '%q' AND value = '%q' "
1699 "AND keypair = '%q' AND signature = '%q' ",
1700 table.c_str(),
1701 t.ident().c_str(),
1702 t.name().c_str(),
1703 t.value().c_str(),
1704 t.key().c_str(),
1705 t.sig().c_str());
1706 I(res.size() == 0 || res.size() == 1);
1707 return res.size() == 1;
1708}
1709
1710void
1711database::put_cert(cert const & t,
1712 string const & table)
1713{
1714 hexenc<id> thash;
1715 cert_hash_code(t, thash);
1716 execute("INSERT INTO '%q' VALUES('%q', '%q', '%q', '%q', '%q', '%q')",
1717 table.c_str(),
1718 thash().c_str(),
1719 t.ident().c_str(),
1720 t.name().c_str(),
1721 t.value().c_str(),
1722 t.key().c_str(),
1723 t.sig().c_str());
1724}
1725
1726void
1727database::results_to_certs(results const & res,
1728 vector<cert> & certs)
1729{
1730 certs.clear();
1731 for (size_t i = 0; i < res.size(); ++i)
1732 {
1733 cert t;
1734 t = cert(hexenc<id>(res[i][0]),
1735 cert_name(res[i][1]),
1736 base64<cert_value>(res[i][2]),
1737 rsa_keypair_id(res[i][3]),
1738 base64<rsa_sha1_signature>(res[i][4]));
1739 certs.push_back(t);
1740 }
1741}
1742
1743void
1744database::install_functions(app_state * app)
1745{
1746 // register any functions we're going to use
1747 I(sqlite3_create_function(sql(), "unbase64", -1,
1748 SQLITE_UTF8, NULL,
1749 &sqlite3_unbase64_fn,
1750 NULL, NULL) == 0);
1751
1752
1753 I(sqlite3_create_function(sql(), "unpack", -1,
1754 SQLITE_UTF8, NULL,
1755 &sqlite3_unpack_fn,
1756 NULL, NULL) == 0);
1757}
1758
1759void
1760database::install_views()
1761{
1762 // we don't currently use any views. re-enable this code if you find a
1763 // compelling reason to use views.
1764
1765 /*
1766 // delete any existing views
1767 results res;
1768 fetch(res, one_col, any_rows,
1769 "SELECT name FROM sqlite_master WHERE type='view'");
1770 for (size_t i = 0; i < res.size(); ++i)
1771 {
1772 execute("DROP VIEW '%q'", res[i][0].c_str());
1773 }
1774 // register any views we're going to use
1775 execute(views_constant);
1776 */
1777}
1778
1779void
1780database::get_certs(vector<cert> & certs,
1781 string const & table)
1782{
1783 results res;
1784 fetch(res, 5, any_rows,
1785 "SELECT id, name, value, keypair, signature FROM '%q' ",
1786 table.c_str());
1787 results_to_certs(res, certs);
1788}
1789
1790
1791void
1792database::get_certs(hexenc<id> const & ident,
1793 vector<cert> & certs,
1794 string const & table)
1795{
1796 results res;
1797 fetch(res, 5, any_rows,
1798 "SELECT id, name, value, keypair, signature FROM '%q' "
1799 "WHERE id = '%q'",
1800 table.c_str(),
1801 ident().c_str());
1802 results_to_certs(res, certs);
1803}
1804
1805
1806void
1807database::get_certs(cert_name const & name,
1808 vector<cert> & certs,
1809 string const & table)
1810{
1811 results res;
1812 fetch(res, 5, any_rows,
1813 "SELECT id, name, value, keypair, signature "
1814 "FROM '%q' WHERE name = '%q'",
1815 table.c_str(),
1816 name().c_str());
1817 results_to_certs(res, certs);
1818}
1819
1820
1821void
1822database::get_certs(hexenc<id> const & ident,
1823 cert_name const & name,
1824 vector<cert> & certs,
1825 string const & table)
1826{
1827 results res;
1828 fetch(res, 5, any_rows,
1829 "SELECT id, name, value, keypair, signature "
1830 "FROM '%q' "
1831 "WHERE id = '%q' AND name = '%q'",
1832 table.c_str(),
1833 ident().c_str(),
1834 name().c_str());
1835 results_to_certs(res, certs);
1836}
1837
1838void
1839database::get_certs(cert_name const & name,
1840 base64<cert_value> const & val,
1841 vector<cert> & certs,
1842 string const & table)
1843{
1844 results res;
1845 fetch(res, 5, any_rows,
1846 "SELECT id, name, value, keypair, signature "
1847 "FROM '%q' "
1848 "WHERE name = '%q' AND value = '%q'",
1849 table.c_str(),
1850 name().c_str(),
1851 val().c_str());
1852 results_to_certs(res, certs);
1853}
1854
1855
1856void
1857database::get_certs(hexenc<id> const & ident,
1858 cert_name const & name,
1859 base64<cert_value> const & value,
1860 vector<cert> & certs,
1861 string const & table)
1862{
1863 results res;
1864 fetch(res, 5, any_rows,
1865 "SELECT id, name, value, keypair, signature "
1866 "FROM '%q' "
1867 "WHERE id = '%q' AND name = '%q' AND value = '%q'",
1868 table.c_str(),
1869 ident().c_str(),
1870 name().c_str(),
1871 value().c_str());
1872 results_to_certs(res, certs);
1873}
1874
1875
1876
1877bool
1878database::revision_cert_exists(revision<cert> const & cert)
1879{
1880 return cert_exists(cert.inner(), "revision_certs");
1881}
1882
1883bool
1884database::manifest_cert_exists(manifest<cert> const & cert)
1885{
1886 return cert_exists(cert.inner(), "manifest_certs");
1887}
1888
1889void
1890database::put_manifest_cert(manifest<cert> const & cert)
1891{
1892 put_cert(cert.inner(), "manifest_certs");
1893}
1894
1895void
1896database::put_revision_cert(revision<cert> const & cert)
1897{
1898 put_cert(cert.inner(), "revision_certs");
1899}
1900
1901void database::get_revision_cert_index(std::vector< std::pair<hexenc<id>,
1902 std::pair<revision_id, rsa_keypair_id> > > & idx)
1903{
1904 results res;
1905 fetch(res, 3, any_rows,
1906 "SELECT hash, id, keypair "
1907 "FROM 'revision_certs'");
1908
1909 idx.clear();
1910 idx.resize(res.size());
1911 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
1912 {
1913 idx.push_back(std::make_pair(hexenc<id>((*i)[0]),
1914 std::make_pair(revision_id((*i)[1]),
1915 rsa_keypair_id((*i)[2]))));
1916 }
1917}
1918
1919void
1920database::get_revision_certs(vector< revision<cert> > & ts)
1921{
1922 vector<cert> certs;
1923 get_certs(certs, "revision_certs");
1924 ts.clear();
1925 copy(certs.begin(), certs.end(), back_inserter(ts));
1926}
1927
1928void
1929database::get_revision_certs(cert_name const & name,
1930 vector< revision<cert> > & ts)
1931{
1932 vector<cert> certs;
1933 get_certs(name, certs, "revision_certs");
1934 ts.clear();
1935 copy(certs.begin(), certs.end(), back_inserter(ts));
1936}
1937
1938void
1939database::get_revision_certs(revision_id const & id,
1940 cert_name const & name,
1941 vector< revision<cert> > & ts)
1942{
1943 vector<cert> certs;
1944 get_certs(id.inner(), name, certs, "revision_certs");
1945 ts.clear();
1946 copy(certs.begin(), certs.end(), back_inserter(ts));
1947}
1948
1949void
1950database::get_revision_certs(revision_id const & id,
1951 cert_name const & name,
1952 base64<cert_value> const & val,
1953 vector< revision<cert> > & ts)
1954{
1955 vector<cert> certs;
1956 get_certs(id.inner(), name, val, certs, "revision_certs");
1957 ts.clear();
1958 copy(certs.begin(), certs.end(), back_inserter(ts));
1959}
1960
1961void
1962database::get_revision_certs(cert_name const & name,
1963 base64<cert_value> const & val,
1964 vector< revision<cert> > & ts)
1965{
1966 vector<cert> certs;
1967 get_certs(name, val, certs, "revision_certs");
1968 ts.clear();
1969 copy(certs.begin(), certs.end(), back_inserter(ts));
1970}
1971
1972void
1973database::get_revision_certs(revision_id const & id,
1974 vector< revision<cert> > & ts)
1975{
1976 vector<cert> certs;
1977 get_certs(id.inner(), certs, "revision_certs");
1978 ts.clear();
1979 copy(certs.begin(), certs.end(), back_inserter(ts));
1980}
1981
1982void
1983database::get_revision_cert(hexenc<id> const & hash,
1984 revision<cert> & c)
1985{
1986 results res;
1987 vector<cert> certs;
1988 fetch(res, 5, one_row,
1989 "SELECT id, name, value, keypair, signature "
1990 "FROM revision_certs "
1991 "WHERE hash = '%q'",
1992 hash().c_str());
1993 results_to_certs(res, certs);
1994 I(certs.size() == 1);
1995 c = revision<cert>(certs[0]);
1996}
1997
1998bool
1999database::revision_cert_exists(hexenc<id> const & hash)
2000{
2001 results res;
2002 vector<cert> certs;
2003 fetch(res, one_col, any_rows,
2004 "SELECT id "
2005 "FROM revision_certs "
2006 "WHERE hash = '%q'",
2007 hash().c_str());
2008 I(res.size() == 0 || res.size() == 1);
2009 return (res.size() == 1);
2010}
2011
2012bool
2013database::manifest_cert_exists(hexenc<id> const & hash)
2014{
2015 results res;
2016 vector<cert> certs;
2017 fetch(res, one_col, any_rows,
2018 "SELECT id "
2019 "FROM manifest_certs "
2020 "WHERE hash = '%q'",
2021 hash().c_str());
2022 I(res.size() == 0 || res.size() == 1);
2023 return (res.size() == 1);
2024}
2025
2026void
2027database::get_manifest_cert(hexenc<id> const & hash,
2028 manifest<cert> & c)
2029{
2030 results res;
2031 vector<cert> certs;
2032 fetch(res, 5, one_row,
2033 "SELECT id, name, value, keypair, signature "
2034 "FROM manifest_certs "
2035 "WHERE hash = '%q'",
2036 hash().c_str());
2037 results_to_certs(res, certs);
2038 I(certs.size() == 1);
2039 c = manifest<cert>(certs[0]);
2040}
2041
2042void
2043database::get_manifest_certs(manifest_id const & id,
2044 vector< manifest<cert> > & ts)
2045{
2046 vector<cert> certs;
2047 get_certs(id.inner(), certs, "manifest_certs");
2048 ts.clear();
2049 copy(certs.begin(), certs.end(), back_inserter(ts));
2050}
2051
2052
2053void
2054database::get_manifest_certs(cert_name const & name,
2055 vector< manifest<cert> > & ts)
2056{
2057 vector<cert> certs;
2058 get_certs(name, certs, "manifest_certs");
2059 ts.clear();
2060 copy(certs.begin(), certs.end(), back_inserter(ts));
2061}
2062
2063void
2064database::get_manifest_certs(manifest_id const & id,
2065 cert_name const & name,
2066 vector< manifest<cert> > & ts)
2067{
2068 vector<cert> certs;
2069 get_certs(id.inner(), name, certs, "manifest_certs");
2070 ts.clear();
2071 copy(certs.begin(), certs.end(), back_inserter(ts));
2072}
2073
2074
2075// completions
2076void
2077database::complete(string const & partial,
2078 set<revision_id> & completions)
2079{
2080 results res;
2081 completions.clear();
2082
2083 fetch(res, 1, any_rows,
2084 "SELECT id FROM revisions WHERE id GLOB '%q*'",
2085 partial.c_str());
2086
2087 for (size_t i = 0; i < res.size(); ++i)
2088 completions.insert(revision_id(res[i][0]));
2089}
2090
2091
2092void
2093database::complete(string const & partial,
2094 set<manifest_id> & completions)
2095{
2096 results res;
2097 completions.clear();
2098
2099 fetch(res, 1, any_rows,
2100 "SELECT id FROM manifests WHERE id GLOB '%q*'",
2101 partial.c_str());
2102
2103 for (size_t i = 0; i < res.size(); ++i)
2104 completions.insert(manifest_id(res[i][0]));
2105
2106 res.clear();
2107
2108 fetch(res, 1, any_rows,
2109 "SELECT id FROM manifest_deltas WHERE id GLOB '%q*'",
2110 partial.c_str());
2111
2112 for (size_t i = 0; i < res.size(); ++i)
2113 completions.insert(manifest_id(res[i][0]));
2114}
2115
2116void
2117database::complete(string const & partial,
2118 set<file_id> & completions)
2119{
2120 results res;
2121 completions.clear();
2122
2123 fetch(res, 1, any_rows,
2124 "SELECT id FROM files WHERE id GLOB '%q*'",
2125 partial.c_str());
2126
2127 for (size_t i = 0; i < res.size(); ++i)
2128 completions.insert(file_id(res[i][0]));
2129
2130 res.clear();
2131
2132 fetch(res, 1, any_rows,
2133 "SELECT id FROM file_deltas WHERE id GLOB '%q*'",
2134 partial.c_str());
2135
2136 for (size_t i = 0; i < res.size(); ++i)
2137 completions.insert(file_id(res[i][0]));
2138}
2139
2140using selectors::selector_type;
2141
2142static void selector_to_certname(selector_type ty,
2143 string & s)
2144{
2145 switch (ty)
2146 {
2147 case selectors::sel_author:
2148 s = author_cert_name;
2149 break;
2150 case selectors::sel_branch:
2151 s = branch_cert_name;
2152 break;
2153 case selectors::sel_date:
2154 s = date_cert_name;
2155 break;
2156 case selectors::sel_tag:
2157 s = tag_cert_name;
2158 break;
2159 case selectors::sel_ident:
2160 case selectors::sel_cert:
2161 case selectors::sel_unknown:
2162 I(false); // don't do this.
2163 break;
2164 }
2165}
2166
2167void database::complete(selector_type ty,
2168 string const & partial,
2169 vector<pair<selector_type, string> > const & limit,
2170 set<string> & completions)
2171{
2172 completions.clear();
2173
2174 // step 1: the limit is transformed into an SQL select statement which
2175 // selects a set of IDs from the manifest_certs table which match the
2176 // limit. this is done by building an SQL select statement for each term
2177 // in the limit and then INTERSECTing them all.
2178
2179 string lim = "(";
2180 if (limit.empty())
2181 {
2182 lim += "SELECT id FROM revision_certs";
2183 }
2184 else
2185 {
2186 bool first_limit = true;
2187 for (vector<pair<selector_type, string> >::const_iterator i = limit.begin();
2188 i != limit.end(); ++i)
2189 {
2190 if (first_limit)
2191 first_limit = false;
2192 else
2193 lim += " INTERSECT ";
2194
2195 if (i->first == selectors::sel_ident)
2196 {
2197 lim += "SELECT id FROM revision_certs ";
2198 lim += (F("WHERE id GLOB '%s*'")
2199 % i->second).str();
2200 }
2201 else if (i->first == selectors::sel_cert)
2202 {
2203 if (i->second.length() > 0)
2204 {
2205 size_t spot = i->second.find("=");
2206
2207 if (spot != (size_t)-1)
2208 {
2209 string certname;
2210 string certvalue;
2211
2212 certname = i->second.substr(0, spot);
2213 spot++;
2214 certvalue = i->second.substr(spot);
2215 lim += "SELECT id FROM revision_certs ";
2216 lim += (F("WHERE name='%s' AND unbase64(value) glob '%s'")
2217 % certname % certvalue).str();
2218 }
2219 else
2220 {
2221 lim += "SELECT id FROM revision_certs ";
2222 lim += (F("WHERE name='%s'")
2223 % i->second).str();
2224 }
2225
2226 }
2227 }
2228 else if (i->first == selectors::sel_unknown)
2229 {
2230 lim += "SELECT id FROM revision_certs ";
2231 lim += (F(" WHERE (name='%s' OR name='%s' OR name='%s')")
2232 % author_cert_name
2233 % tag_cert_name
2234 % branch_cert_name).str();
2235 lim += (F(" AND unbase64(value) glob '*%s*'")
2236 % i->second).str();
2237 }
2238 else
2239 {
2240 string certname;
2241 selector_to_certname(i->first, certname);
2242 lim += "SELECT id FROM revision_certs ";
2243 lim += (F("WHERE name='%s' AND unbase64(value) glob '*%s*'")
2244 % certname % i->second).str();
2245 }
2246 }
2247 }
2248 lim += ")";
2249
2250 // step 2: depending on what we've been asked to disambiguate, we
2251 // will complete either some idents, or cert values, or "unknown"
2252 // which generally means "author, tag or branch"
2253
2254 string query;
2255 if (ty == selectors::sel_ident)
2256 {
2257 query = (F("SELECT id FROM %s") % lim).str();
2258 }
2259 else
2260 {
2261 query = "SELECT value FROM revision_certs WHERE";
2262 if (ty == selectors::sel_unknown)
2263 {
2264 query +=
2265 (F(" (name='%s' OR name='%s' OR name='%s')")
2266 % author_cert_name
2267 % tag_cert_name
2268 % branch_cert_name).str();
2269 }
2270 else
2271 {
2272 string certname;
2273 selector_to_certname(ty, certname);
2274 query +=
2275 (F(" (name='%s')") % certname).str();
2276 }
2277
2278 query += (F(" AND (unbase64(value) GLOB '*%s*')") % partial).str();
2279 query += (F(" AND (id IN %s)") % lim).str();
2280 }
2281
2282 results res;
2283 fetch(res, one_col, any_rows, query.c_str());
2284 for (size_t i = 0; i < res.size(); ++i)
2285 {
2286 if (ty == selectors::sel_ident)
2287 completions.insert(res[i][0]);
2288 else
2289 {
2290 base64<data> row_encoded(res[i][0]);
2291 data row_decoded;
2292 decode_base64(row_encoded, row_decoded);
2293 completions.insert(row_decoded());
2294 }
2295 }
2296}
2297
2298// epochs
2299
2300void
2301database::get_epochs(std::map<cert_value, epoch_data> & epochs)
2302{
2303 epochs.clear();
2304 results res;
2305 fetch(res, 2, any_rows, "SELECT branch, epoch FROM branch_epochs");
2306 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
2307 {
2308 base64<cert_value> encoded(idx(*i, 0));
2309 cert_value decoded;
2310 decode_base64(encoded, decoded);
2311 I(epochs.find(decoded) == epochs.end());
2312 epochs.insert(std::make_pair(decoded, epoch_data(idx(*i, 1))));
2313 }
2314}
2315
2316void
2317database::get_epoch(epoch_id const & eid,
2318 cert_value & branch, epoch_data & epo)
2319{
2320 I(epoch_exists(eid));
2321 results res;
2322 fetch(res, 2, any_rows,
2323 "SELECT branch, epoch FROM branch_epochs"
2324 " WHERE hash = '%q'",
2325 eid.inner()().c_str());
2326 I(res.size() == 1);
2327 base64<cert_value> encoded(idx(idx(res, 0), 0));
2328 decode_base64(encoded, branch);
2329 epo = epoch_data(idx(idx(res, 0), 1));
2330}
2331
2332bool
2333database::epoch_exists(epoch_id const & eid)
2334{
2335 results res;
2336 fetch(res, one_col, any_rows,
2337 "SELECT hash FROM branch_epochs WHERE hash = '%q'",
2338 eid.inner()().c_str());
2339 I(res.size() == 1 || res.size() == 0);
2340 return res.size() == 1;
2341}
2342
2343void
2344database::set_epoch(cert_value const & branch, epoch_data const & epo)
2345{
2346 epoch_id eid;
2347 base64<cert_value> encoded;
2348 encode_base64(branch, encoded);
2349 epoch_hash_code(branch, epo, eid);
2350 I(epo.inner()().size() == constants::epochlen);
2351 execute("INSERT OR REPLACE INTO branch_epochs VALUES('%q', '%q', '%q')",
2352 eid.inner()().c_str(), encoded().c_str(), epo.inner()().c_str());
2353}
2354
2355void
2356database::clear_epoch(cert_value const & branch)
2357{
2358 base64<cert_value> encoded;
2359 encode_base64(branch, encoded);
2360 execute("DELETE FROM branch_epochs WHERE branch = '%q'", encoded().c_str());
2361}
2362
2363// vars
2364
2365void
2366database::get_vars(std::map<var_key, var_value> & vars)
2367{
2368 vars.clear();
2369 results res;
2370 fetch(res, 3, any_rows, "SELECT domain, name, value FROM db_vars");
2371 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
2372 {
2373 var_domain domain(idx(*i, 0));
2374 base64<var_name> name_encoded(idx(*i, 1));
2375 var_name name;
2376 decode_base64(name_encoded, name);
2377 base64<var_value> value_encoded(idx(*i, 2));
2378 var_value value;
2379 decode_base64(value_encoded, value);
2380 I(vars.find(std::make_pair(domain, name)) == vars.end());
2381 vars.insert(std::make_pair(std::make_pair(domain, name), value));
2382 }
2383}
2384
2385void
2386database::get_var(var_key const & key, var_value & value)
2387{
2388 // FIXME: sillyly inefficient. Doesn't really matter, though.
2389 std::map<var_key, var_value> vars;
2390 get_vars(vars);
2391 std::map<var_key, var_value>::const_iterator i = vars.find(key);
2392 I(i != vars.end());
2393 value = i->second;
2394}
2395
2396bool
2397database::var_exists(var_key const & key)
2398{
2399 // FIXME: sillyly inefficient. Doesn't really matter, though.
2400 std::map<var_key, var_value> vars;
2401 get_vars(vars);
2402 std::map<var_key, var_value>::const_iterator i = vars.find(key);
2403 return i != vars.end();
2404}
2405
2406void
2407database::set_var(var_key const & key, var_value const & value)
2408{
2409 base64<var_name> name_encoded;
2410 encode_base64(key.second, name_encoded);
2411 base64<var_value> value_encoded;
2412 encode_base64(value, value_encoded);
2413 execute("INSERT OR REPLACE INTO db_vars VALUES('%q', '%q', '%q')",
2414 key.first().c_str(),
2415 name_encoded().c_str(),
2416 value_encoded().c_str());
2417}
2418
2419void
2420database::clear_var(var_key const & key)
2421{
2422 base64<var_name> name_encoded;
2423 encode_base64(key.second, name_encoded);
2424 execute("DELETE FROM db_vars WHERE domain = '%q' AND name = '%q'",
2425 key.first().c_str(), name_encoded().c_str());
2426}
2427
2428// transaction guards
2429
2430transaction_guard::transaction_guard(database & d) : committed(false), db(d)
2431{
2432 db.begin_transaction();
2433}
2434transaction_guard::~transaction_guard()
2435{
2436 if (committed)
2437 db.commit_transaction();
2438 else
2439 db.rollback_transaction();
2440}
2441
2442void
2443transaction_guard::commit()
2444{
2445 committed = true;
2446}
2447

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status