monotone

monotone Mtn Source Tree

Root/database.cc

1// Copyright (C) 2002 Graydon Hoare <graydon@pobox.com>
2//
3// This program is made available under the GNU GPL version 2.0 or
4// greater. See the accompanying file COPYING for details.
5//
6// This program is distributed WITHOUT ANY WARRANTY; without even the
7// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
8// PURPOSE.
9
10#include "base.hh"
11#include <algorithm>
12#include <deque>
13#include <fstream>
14#include <iterator>
15#include <list>
16#include <set>
17#include <sstream>
18#include "vector.hh"
19
20#include <string.h>
21
22#include <boost/shared_ptr.hpp>
23#include "lexical_cast.hh"
24
25#include "sqlite/sqlite3.h"
26
27#include "app_state.hh"
28#include "cert.hh"
29#include "cleanup.hh"
30#include "constants.hh"
31#include "dates.hh"
32#include "database.hh"
33#include "hash_map.hh"
34#include "keys.hh"
35#include "platform-wrapped.hh"
36#include "revision.hh"
37#include "safe_map.hh"
38#include "sanity.hh"
39#include "schema_migration.hh"
40#include "transforms.hh"
41#include "ui.hh"
42#include "vocab.hh"
43#include "vocab_cast.hh"
44#include "xdelta.hh"
45#include "epoch.hh"
46#include "graph.hh"
47#include "roster.hh"
48#include "roster_delta.hh"
49#include "rev_height.hh"
50#include "vocab_hash.hh"
51#include "globish.hh"
52#include "work.hh"
53#include "lua_hooks.hh"
54#include "outdated_indicator.hh"
55#include "lru_writeback_cache.hh"
56
57#include "botan/botan.h"
58#include "botan/rsa.h"
59#include "botan/keypair.h"
60#include "botan/pem.h"
61
62// defined in schema.c, generated from schema.sql:
63extern char const schema_constant[];
64
65// this file defines a public, typed interface to the database.
66// the database class encapsulates all knowledge about sqlite,
67// the schema, and all SQL statements used to access the schema.
68//
69// see file schema.sql for the text of the schema.
70
71using std::deque;
72using std::istream;
73using std::ifstream;
74using std::make_pair;
75using std::map;
76using std::multimap;
77using std::ostream;
78using std::pair;
79using std::set;
80using std::string;
81using std::vector;
82
83using boost::shared_ptr;
84using boost::shared_dynamic_cast;
85using boost::lexical_cast;
86
87using Botan::PK_Encryptor;
88using Botan::PK_Verifier;
89using Botan::SecureVector;
90using Botan::X509_PublicKey;
91using Botan::RSA_PublicKey;
92
93int const one_row = 1;
94int const one_col = 1;
95int const any_rows = -1;
96int const any_cols = -1;
97
98namespace
99{
100 struct query_param
101 {
102 enum arg_type { text, blob };
103 arg_type type;
104 string data;
105 };
106
107 query_param
108 text(string const & txt)
109 {
110 query_param q = {
111 query_param::text,
112 txt,
113 };
114 return q;
115 }
116
117 query_param
118 blob(string const & blb)
119 {
120 query_param q = {
121 query_param::blob,
122 blb,
123 };
124 return q;
125 }
126
127 struct query
128 {
129 explicit query(string const & cmd)
130 : sql_cmd(cmd)
131 {}
132
133 query()
134 {}
135
136 query & operator %(query_param const & qp)
137 {
138 args.push_back(qp);
139 return *this;
140 }
141
142 vector<query_param> args;
143 string sql_cmd;
144 };
145
146 typedef vector< vector<string> > results;
147
148 struct statement
149 {
150 statement() : count(0), stmt(0, sqlite3_finalize) {}
151 int count;
152 cleanup_ptr<sqlite3_stmt*, int> stmt;
153 };
154
155 struct roster_size_estimator
156 {
157 unsigned long operator() (cached_roster const & cr)
158 {
159 I(cr.first);
160 I(cr.second);
161 // do estimate using a totally made up multiplier, probably wildly off
162 return (cr.first->all_nodes().size()
163 * constants::db_estimated_roster_node_sz);
164 }
165 };
166
167 struct datasz
168 {
169 unsigned long operator()(data const & t) { return t().size(); }
170 };
171
172 enum open_mode { normal_mode = 0,
173 schema_bypass_mode,
174 format_bypass_mode };
175
176 typedef hashmap::hash_map<revision_id, set<revision_id> > parent_id_map;
177 typedef hashmap::hash_map<revision_id, rev_height> height_map;
178
179 typedef hashmap::hash_map<rsa_keypair_id,
180 pair<shared_ptr<Botan::PK_Verifier>,
181 shared_ptr<Botan::RSA_PublicKey> >
182 > verifier_cache;
183
184} // anonymous namespace
185
186class database_impl
187{
188 friend class database;
189
190 // for scoped_ptr's sake
191public:
192 explicit database_impl(system_path const &);
193 ~database_impl();
194
195private:
196
197 //
198 // --== Opening the database and schema checking ==--
199 //
200 system_path const filename;
201 struct sqlite3 * __sql;
202
203 void install_functions();
204 struct sqlite3 * sql(enum open_mode mode = normal_mode);
205
206 void check_filename();
207 void check_db_exists();
208 void check_db_nonexistent();
209 void open();
210 void close();
211 void check_format();
212
213 //
214 // --== Basic SQL interface and statement caching ==--
215 //
216 map<string, statement> statement_cache;
217
218 void fetch(results & res,
219 int const want_cols, int const want_rows,
220 query const & q);
221 void execute(query const & q);
222
223 bool table_has_entry(id const & key, string const & column,
224 string const & table);
225
226 //
227 // --== Generic database metadata gathering ==--
228 //
229 string count(string const & table);
230 string space(string const & table,
231 string const & concatenated_columns,
232 u64 & total);
233 unsigned int page_size();
234 unsigned int cache_size();
235
236 //
237 // --== Transactions ==--
238 //
239 int transaction_level;
240 bool transaction_exclusive;
241 void begin_transaction(bool exclusive);
242 void commit_transaction();
243 void rollback_transaction();
244 friend class conditional_transaction_guard;
245
246 struct roster_writeback_manager
247 {
248 database_impl & imp;
249 roster_writeback_manager(database_impl & imp) : imp(imp) {}
250 void writeout(revision_id const &, cached_roster const &);
251 };
252 LRUWritebackCache<revision_id, cached_roster,
253 roster_size_estimator, roster_writeback_manager>
254 roster_cache;
255
256 bool have_delayed_file(file_id const & id);
257 void load_delayed_file(file_id const & id, file_data & dat);
258 void cancel_delayed_file(file_id const & id);
259 void drop_or_cancel_file(file_id const & id);
260 void schedule_delayed_file(file_id const & id, file_data const & dat);
261
262 map<file_id, file_data> delayed_files;
263 size_t delayed_writes_size;
264
265 void flush_delayed_writes();
266 void clear_delayed_writes();
267 void write_delayed_file(file_id const & new_id,
268 file_data const & dat);
269
270 void write_delayed_roster(revision_id const & new_id,
271 roster_t const & roster,
272 marking_map const & marking);
273
274 //
275 // --== Reading/writing delta-compressed objects ==--
276 //
277
278 // "do we have any entry for 'ident' that is a base version"
279 bool roster_base_stored(revision_id const & ident);
280 bool roster_base_available(revision_id const & ident);
281
282 // "do we have any entry for 'ident' that is a delta"
283 bool delta_exists(file_id const & ident,
284 file_id const & base,
285 string const & table);
286
287 bool file_or_manifest_base_exists(id const & ident,
288 std::string const & table);
289
290 void get_file_or_manifest_base_unchecked(id const & new_id,
291 data & dat,
292 string const & table);
293 void get_file_or_manifest_delta_unchecked(id const & ident,
294 id const & base,
295 delta & del,
296 string const & table);
297 void get_roster_base(revision_id const & ident,
298 roster_t & roster, marking_map & marking);
299 void get_roster_delta(id const & ident,
300 id const & base,
301 roster_delta & del);
302
303 friend struct file_and_manifest_reconstruction_graph;
304 friend struct roster_reconstruction_graph;
305
306 LRUWritebackCache<id, data, datasz> vcache;
307
308 void get_version(id const & ident,
309 data & dat,
310 string const & data_table,
311 string const & delta_table);
312
313 void drop(id const & base,
314 string const & table);
315 void put_file_delta(file_id const & ident,
316 file_id const & base,
317 file_delta const & del);
318
319 void put_roster_delta(revision_id const & ident,
320 revision_id const & base,
321 roster_delta const & del);
322 void put_version(file_id const & old_id,
323 file_id const & new_id,
324 delta const & del,
325 string const & data_table,
326 string const & delta_table);
327
328 //
329 // --== The ancestry graph ==--
330 //
331 void get_ids(string const & table, set<id> & ids);
332
333 //
334 // --== Rosters ==--
335 //
336 struct extractor;
337 struct file_content_extractor;
338 struct markings_extractor;
339 void extract_from_deltas(revision_id const & id, extractor & x);
340
341 height_map height_cache;
342 parent_id_map parent_cache;
343
344 //
345 // --== Keys ==--
346 //
347 void get_keys(string const & table, vector<rsa_keypair_id> & keys);
348
349 // cache of verifiers for public keys
350 verifier_cache verifiers;
351
352 //
353 // --== Certs ==--
354 //
355 // note: this section is ridiculous. please do something about it.
356 bool cert_exists(cert const & t,
357 string const & table);
358 void put_cert(cert const & t, string const & table);
359 void results_to_certs(results const & res,
360 vector<cert> & certs);
361
362 void get_certs(vector<cert> & certs,
363 string const & table);
364
365 void get_certs(id const & ident,
366 vector<cert> & certs,
367 string const & table);
368
369 void get_certs(cert_name const & name,
370 vector<cert> & certs,
371 string const & table);
372
373 void get_certs(id const & ident,
374 cert_name const & name,
375 vector<cert> & certs,
376 string const & table);
377
378 void get_certs(id const & ident,
379 cert_name const & name,
380 cert_value const & val,
381 vector<cert> & certs,
382 string const & table);
383
384 void get_certs(cert_name const & name,
385 cert_value const & val,
386 vector<cert> & certs,
387 string const & table);
388
389 outdated_indicator_factory cert_stamper;
390
391 void add_prefix_matching_constraint(string const & colname,
392 string const & prefix,
393 query & q);
394};
395
396database_impl::database_impl(system_path const & f) :
397 filename(f),
398 __sql(NULL),
399 transaction_level(0),
400 roster_cache(constants::db_roster_cache_sz,
401 roster_writeback_manager(*this)),
402 delayed_writes_size(0),
403 vcache(constants::db_version_cache_sz)
404{}
405
406database_impl::~database_impl()
407{
408 L(FL("statement cache statistics"));
409 L(FL("prepared %d statements") % statement_cache.size());
410
411 for (map<string, statement>::const_iterator i = statement_cache.begin();
412 i != statement_cache.end(); ++i)
413 L(FL("%d executions of %s") % i->second.count % i->first);
414 // trigger destructors to finalize cached statements
415 statement_cache.clear();
416
417 if (__sql)
418 close();
419}
420
421database::database(app_state & app)
422 : lua(app.lua), rng(app.rng)
423{
424 boost::shared_ptr<database_impl> & i = app.lookup_db(app.opts.dbname);
425 if (!i)
426 {
427 i.reset(new database_impl(app.opts.dbname));
428 }
429 imp = i;
430}
431
432database::~database()
433{}
434
435system_path
436database::get_filename()
437{
438 return imp->filename;
439}
440
441bool
442database::is_dbfile(any_path const & file)
443{
444 system_path fn(file); // canonicalize
445 bool same = (imp->filename == fn);
446 if (same)
447 L(FL("'%s' is the database file") % file);
448 return same;
449}
450
451bool
452database::database_specified()
453{
454 return !imp->filename.empty();
455}
456
457void
458database::check_is_not_rosterified()
459{
460 results res;
461 imp->fetch(res, one_col, any_rows,
462 query("SELECT 1 FROM rosters LIMIT 1"));
463 N(res.empty(),
464 F("this database already contains rosters"));
465}
466
467void
468database_impl::check_format()
469{
470 results res;
471
472 // Check for manifests, revisions, rosters, and heights.
473 fetch(res, one_col, any_rows, query("SELECT 1 FROM manifests LIMIT 1"));
474 bool have_manifests = !res.empty();
475 fetch(res, one_col, any_rows, query("SELECT 1 FROM revisions LIMIT 1"));
476 bool have_revisions = !res.empty();
477 fetch(res, one_col, any_rows, query("SELECT 1 FROM rosters LIMIT 1"));
478 bool have_rosters = !res.empty();
479 fetch(res, one_col, any_rows, query("SELECT 1 FROM heights LIMIT 1"));
480 bool have_heights = !res.empty();
481
482
483 if (!have_manifests)
484 {
485 // Must have been changesetified and rosterified already.
486 // Or else the database was just created.
487 // Do we need to regenerate cached data?
488 E(!have_revisions || (have_rosters && have_heights),
489 F("database %s lacks some cached data\n"
490 "run '%s db regenerate_caches' to restore use of this database")
491 % filename % ui.prog_name);
492 }
493 else
494 {
495 // The rosters and heights tables should be empty.
496 I(!have_rosters && !have_heights);
497
498 // they need to either changesetify or rosterify. which?
499 if (have_revisions)
500 E(false,
501 F("database %s contains old-style revisions\n"
502 "if you are a project leader or doing local testing:\n"
503 " see the file UPGRADE for instructions on upgrading.\n"
504 "if you are not a project leader:\n"
505 " wait for a leader to migrate project data, and then\n"
506 " pull into a fresh database.\n"
507 "sorry about the inconvenience.")
508 % filename);
509 else
510 E(false,
511 F("database %s contains manifests but no revisions\n"
512 "this is a very old database; it needs to be upgraded\n"
513 "please see README.changesets for details")
514 % filename);
515 }
516}
517
518static void
519sqlite3_gunzip_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
520{
521 if (nargs != 1)
522 {
523 sqlite3_result_error(f, "need exactly 1 arg to gunzip()", -1);
524 return;
525 }
526 data unpacked;
527 const char *val = (const char*) sqlite3_value_blob(args[0]);
528 int bytes = sqlite3_value_bytes(args[0]);
529 decode_gzip(gzip<data>(string(val,val+bytes)), unpacked);
530 sqlite3_result_blob(f, unpacked().c_str(), unpacked().size(), SQLITE_TRANSIENT);
531}
532
533struct sqlite3 *
534database_impl::sql(enum open_mode mode)
535{
536 if (! __sql)
537 {
538 check_filename();
539 check_db_exists();
540 open();
541
542 if (mode != schema_bypass_mode)
543 {
544 check_sql_schema(__sql, filename);
545
546 if (mode != format_bypass_mode)
547 check_format();
548 }
549
550 install_functions();
551 }
552 else
553 I(mode == normal_mode);
554
555 return __sql;
556}
557
558void
559database::initialize()
560{
561 imp->check_filename();
562 imp->check_db_nonexistent();
563 imp->open();
564
565 sqlite3 *sql = imp->__sql;
566
567 sqlite3_exec(sql, schema_constant, NULL, NULL, NULL);
568 assert_sqlite3_ok(sql);
569
570 sqlite3_exec(sql, (FL("PRAGMA user_version = %u;")
571 % mtn_creator_code).str().c_str(), NULL, NULL, NULL);
572 assert_sqlite3_ok(sql);
573
574 // make sure what we wanted is what we got
575 check_sql_schema(sql, imp->filename);
576
577 imp->close();
578}
579
580struct
581dump_request
582{
583 dump_request() : sql(), out() {};
584 struct sqlite3 *sql;
585 ostream *out;
586};
587
588static void
589dump_row(ostream &out, sqlite3_stmt *stmt, string const& table_name)
590{
591 out << FL("INSERT INTO %s VALUES(") % table_name;
592 unsigned n = sqlite3_data_count(stmt);
593 for (unsigned i = 0; i < n; ++i)
594 {
595 if (i != 0)
596 out << ',';
597
598 if (sqlite3_column_type(stmt, i) == SQLITE_BLOB)
599 {
600 out << "X'";
601 const char *val = (const char*) sqlite3_column_blob(stmt, i);
602 int bytes = sqlite3_column_bytes(stmt, i);
603 out << encode_hexenc(string(val,val+bytes));
604 out << '\'';
605 }
606 else
607 {
608 const unsigned char *val = sqlite3_column_text(stmt, i);
609 if (val == NULL)
610 out << "NULL";
611 else
612 {
613 out << '\'';
614 for (const unsigned char *cp = val; *cp; ++cp)
615 {
616 if (*cp == '\'')
617 out << "''";
618 else
619 out << *cp;
620 }
621 out << '\'';
622 }
623 }
624 }
625 out << ");\n";
626}
627
628static int
629dump_table_cb(void *data, int n, char **vals, char **cols)
630{
631 dump_request *dump = reinterpret_cast<dump_request *>(data);
632 I(dump != NULL);
633 I(dump->sql != NULL);
634 I(vals != NULL);
635 I(vals[0] != NULL);
636 I(vals[1] != NULL);
637 I(vals[2] != NULL);
638 I(n == 3);
639 I(string(vals[1]) == "table");
640 *(dump->out) << vals[2] << ";\n";
641 string table_name(vals[0]);
642 string query = "SELECT * FROM " + table_name;
643 sqlite3_stmt *stmt = 0;
644 sqlite3_prepare_v2(dump->sql, query.c_str(), -1, &stmt, NULL);
645 assert_sqlite3_ok(dump->sql);
646
647 int stepresult = SQLITE_DONE;
648 do
649 {
650 stepresult = sqlite3_step(stmt);
651 I(stepresult == SQLITE_DONE || stepresult == SQLITE_ROW);
652 if (stepresult == SQLITE_ROW)
653 dump_row(*(dump->out), stmt, table_name);
654 }
655 while (stepresult == SQLITE_ROW);
656
657 sqlite3_finalize(stmt);
658 assert_sqlite3_ok(dump->sql);
659 return 0;
660}
661
662static int
663dump_index_cb(void *data, int n, char **vals, char **cols)
664{
665 dump_request *dump = reinterpret_cast<dump_request *>(data);
666 I(dump != NULL);
667 I(dump->sql != NULL);
668 I(vals != NULL);
669 I(vals[0] != NULL);
670 I(vals[1] != NULL);
671 I(vals[2] != NULL);
672 I(n == 3);
673 I(string(vals[1]) == "index");
674 *(dump->out) << vals[2] << ";\n";
675 return 0;
676}
677
678static int
679dump_user_version_cb(void *data, int n, char **vals, char **cols)
680{
681 dump_request *dump = reinterpret_cast<dump_request *>(data);
682 I(dump != NULL);
683 I(dump->sql != NULL);
684 I(vals != NULL);
685 I(vals[0] != NULL);
686 I(n == 1);
687 *(dump->out) << "PRAGMA user_version = " << vals[0] << ";\n";
688 return 0;
689}
690
691void
692database::dump(ostream & out)
693{
694 ensure_open_for_maintenance();
695
696 {
697 transaction_guard guard(*this);
698 dump_request req;
699 req.out = &out;
700 req.sql = imp->sql();
701 out << "BEGIN EXCLUSIVE;\n";
702 int res;
703 res = sqlite3_exec(req.sql,
704 "SELECT name, type, sql FROM sqlite_master "
705 "WHERE type='table' AND sql NOT NULL "
706 "AND name not like 'sqlite_stat%' "
707 "ORDER BY name",
708 dump_table_cb, &req, NULL);
709 assert_sqlite3_ok(req.sql);
710 res = sqlite3_exec(req.sql,
711 "SELECT name, type, sql FROM sqlite_master "
712 "WHERE type='index' AND sql NOT NULL "
713 "ORDER BY name",
714 dump_index_cb, &req, NULL);
715 assert_sqlite3_ok(req.sql);
716 res = sqlite3_exec(req.sql,
717 "PRAGMA user_version;",
718 dump_user_version_cb, &req, NULL);
719 assert_sqlite3_ok(req.sql);
720 out << "COMMIT;\n";
721 guard.commit();
722 }
723}
724
725void
726database::load(istream & in)
727{
728 string line;
729 string sql_stmt;
730
731 imp->check_filename();
732 imp->check_db_nonexistent();
733 imp->open();
734
735 sqlite3 * sql = imp->__sql;
736
737 // the page size can only be set before any other commands have been executed
738 sqlite3_exec(sql, "PRAGMA page_size=8192", NULL, NULL, NULL);
739 assert_sqlite3_ok(sql);
740
741 while(in)
742 {
743 getline(in, line, ';');
744 sql_stmt += line + ';';
745
746 if (sqlite3_complete(sql_stmt.c_str()))
747 {
748 sqlite3_exec(sql, sql_stmt.c_str(), NULL, NULL, NULL);
749 assert_sqlite3_ok(sql);
750 sql_stmt.clear();
751 }
752 }
753
754 assert_sqlite3_ok(sql);
755}
756
757
758void
759database::debug(string const & sql, ostream & out)
760{
761 ensure_open_for_maintenance();
762
763 results res;
764 imp->fetch(res, any_cols, any_rows, query(sql));
765 out << '\'' << sql << "' -> " << res.size() << " rows\n\n";
766 for (size_t i = 0; i < res.size(); ++i)
767 {
768 for (size_t j = 0; j < res[i].size(); ++j)
769 {
770 if (j != 0)
771 out << " | ";
772 out << res[i][j];
773 }
774 out << '\n';
775 }
776}
777
778// Subroutine of info(). This compares strings that might either be numbers
779// or error messages surrounded by square brackets. We want the longest
780// number, even if there's an error message that's longer than that.
781static bool longest_number(string a, string b)
782{
783 if(a.length() > 0 && a[0] == '[')
784 return true; // b is longer
785 if(b.length() > 0 && b[0] == '[')
786 return false; // a is longer
787
788 return a.length() < b.length();
789}
790
791// Subroutine of info() and some things it calls.
792// Given an informative_failure which is believed to represent an SQLite
793// error, either return a string version of the error message (if it was an
794// SQLite error) or rethrow the execption (if it wasn't).
795static string
796format_sqlite_error_for_info(informative_failure const & e)
797{
798 string err(e.what());
799 string prefix = _("error: ");
800 prefix.append(_("sqlite error: "));
801 if (err.find(prefix) != 0)
802 throw;
803
804 err.replace(0, prefix.length(), "[");
805 string::size_type nl = err.find('\n');
806 if (nl != string::npos)
807 err.erase(nl);
808
809 err.append("]");
810 return err;
811}
812
813// Subroutine of info(). Pretty-print the database's "creator code", which
814// is a 32-bit unsigned number that we interpret as a four-character ASCII
815// string, provided that all four characters are graphic. (On disk, it's
816// stored in the "user version" field of the database.)
817static string
818format_creator_code(u32 code)
819{
820 char buf[5];
821 string result;
822
823 if (code == 0)
824 return _("not set");
825
826 buf[4] = '\0';
827 buf[3] = ((code & 0x000000ff) >> 0);
828 buf[2] = ((code & 0x0000ff00) >> 8);
829 buf[1] = ((code & 0x00ff0000) >> 16);
830 buf[0] = ((code & 0xff000000) >> 24);
831
832 if (isgraph(buf[0]) && isgraph(buf[1]) && isgraph(buf[2]) && isgraph(buf[3]))
833 result = (FL("%s (0x%08x)") % buf % code).str();
834 else
835 result = (FL("0x%08x") % code).str();
836 if (code != mtn_creator_code)
837 result += _(" (not a monotone database)");
838 return result;
839}
840
841
842void
843database::info(ostream & out)
844{
845 // don't check the schema
846 ensure_open_for_maintenance();
847
848 // do a dummy query to confirm that the database file is an sqlite3
849 // database. (this doesn't happen on open() because sqlite postpones the
850 // actual file open until the first access. we can't piggyback it on the
851 // query of the user version because there's a bug in sqlite 3.3.10:
852 // the routine that reads meta-values from the database header does not
853 // check the file format. reported as sqlite bug #2182.)
854 sqlite3_exec(imp->__sql, "SELECT 1 FROM sqlite_master LIMIT 0", 0, 0, 0);
855 assert_sqlite3_ok(imp->__sql);
856
857 u32 ccode;
858 {
859 results res;
860 imp->fetch(res, one_col, one_row, query("PRAGMA user_version"));
861 I(res.size() == 1);
862 ccode = lexical_cast<u32>(res[0][0]);
863 }
864
865 vector<string> counts;
866 counts.push_back(imp->count("rosters"));
867 counts.push_back(imp->count("roster_deltas"));
868 counts.push_back(imp->count("files"));
869 counts.push_back(imp->count("file_deltas"));
870 counts.push_back(imp->count("revisions"));
871 counts.push_back(imp->count("revision_ancestry"));
872 counts.push_back(imp->count("revision_certs"));
873
874 {
875 results res;
876 try
877 {
878 imp->fetch(res, one_col, any_rows,
879 query("SELECT node FROM next_roster_node_number"));
880 if (res.empty())
881 counts.push_back("0");
882 else
883 {
884 I(res.size() == 1);
885 u64 n = lexical_cast<u64>(res[0][0]) - 1;
886 counts.push_back((F("%u") % n).str());
887 }
888 }
889 catch (informative_failure const & e)
890 {
891 counts.push_back(format_sqlite_error_for_info(e));
892 }
893 }
894
895 vector<string> bytes;
896 {
897 u64 total = 0;
898 bytes.push_back(imp->space("rosters",
899 "length(id) + length(checksum) + length(data)",
900 total));
901 bytes.push_back(imp->space("roster_deltas",
902 "length(id) + length(checksum)"
903 "+ length(base) + length(delta)", total));
904 bytes.push_back(imp->space("files", "length(id) + length(data)", total));
905 bytes.push_back(imp->space("file_deltas",
906 "length(id) + length(base) + length(delta)", total));
907 bytes.push_back(imp->space("revisions", "length(id) + length(data)", total));
908 bytes.push_back(imp->space("revision_ancestry",
909 "length(parent) + length(child)", total));
910 bytes.push_back(imp->space("revision_certs",
911 "length(hash) + length(id) + length(name)"
912 "+ length(value) + length(keypair)"
913 "+ length(signature)", total));
914 bytes.push_back(imp->space("heights", "length(revision) + length(height)",
915 total));
916 bytes.push_back((F("%u") % total).str());
917 }
918
919 // pad each vector's strings on the left with spaces to make them all the
920 // same length
921 {
922 string::size_type width
923 = max_element(counts.begin(), counts.end(), longest_number)->length();
924 for(vector<string>::iterator i = counts.begin(); i != counts.end(); i++)
925 if (width > i->length() && (*i)[0] != '[')
926 i->insert(0, width - i->length(), ' ');
927
928 width = max_element(bytes.begin(), bytes.end(), longest_number)->length();
929 for(vector<string>::iterator i = bytes.begin(); i != bytes.end(); i++)
930 if (width > i->length() && (*i)[0] != '[')
931 i->insert(0, width - i->length(), ' ');
932 }
933
934 i18n_format form =
935 F("creator code : %s\n"
936 "schema version : %s\n"
937 "counts:\n"
938 " full rosters : %s\n"
939 " roster deltas : %s\n"
940 " full files : %s\n"
941 " file deltas : %s\n"
942 " revisions : %s\n"
943 " ancestry edges : %s\n"
944 " certs : %s\n"
945 " logical files : %s\n"
946 "bytes:\n"
947 " full rosters : %s\n"
948 " roster deltas : %s\n"
949 " full files : %s\n"
950 " file deltas : %s\n"
951 " revisions : %s\n"
952 " cached ancestry : %s\n"
953 " certs : %s\n"
954 " heights : %s\n"
955 " total : %s\n"
956 "database:\n"
957 " page size : %s\n"
958 " cache size : %s"
959 );
960
961 form = form % format_creator_code(ccode);
962 form = form % describe_sql_schema(imp->__sql);
963
964 for (vector<string>::iterator i = counts.begin(); i != counts.end(); i++)
965 form = form % *i;
966
967 for (vector<string>::iterator i = bytes.begin(); i != bytes.end(); i++)
968 form = form % *i;
969
970 form = form % imp->page_size();
971 form = form % imp->cache_size();
972
973 out << form.str() << '\n'; // final newline is kept out of the translation
974}
975
976void
977database::version(ostream & out)
978{
979 ensure_open_for_maintenance();
980 out << (F("database schema version: %s")
981 % describe_sql_schema(imp->__sql)).str()
982 << '\n';
983}
984
985void
986database::migrate(key_store & keys)
987{
988 ensure_open_for_maintenance();
989 migrate_sql_schema(imp->__sql, keys, get_filename());
990}
991
992void
993database::test_migration_step(key_store & keys, string const & schema)
994{
995 ensure_open_for_maintenance();
996 ::test_migration_step(imp->__sql, keys, get_filename(), schema);
997}
998
999void
1000database::ensure_open()
1001{
1002 imp->sql();
1003}
1004
1005void
1006database::ensure_open_for_format_changes()
1007{
1008 imp->sql(format_bypass_mode);
1009}
1010
1011void
1012database::ensure_open_for_maintenance()
1013{
1014 imp->sql(schema_bypass_mode);
1015}
1016
1017void
1018database_impl::execute(query const & query)
1019{
1020 results res;
1021 fetch(res, 0, 0, query);
1022}
1023
1024void
1025database_impl::fetch(results & res,
1026 int const want_cols,
1027 int const want_rows,
1028 query const & query)
1029{
1030 int nrow;
1031 int ncol;
1032 int rescode;
1033
1034 res.clear();
1035 res.resize(0);
1036
1037 map<string, statement>::iterator i = statement_cache.find(query.sql_cmd);
1038 if (i == statement_cache.end())
1039 {
1040 statement_cache.insert(make_pair(query.sql_cmd, statement()));
1041 i = statement_cache.find(query.sql_cmd);
1042 I(i != statement_cache.end());
1043
1044 const char * tail;
1045 sqlite3_prepare_v2(sql(), query.sql_cmd.c_str(), -1, i->second.stmt.paddr(), &tail);
1046 assert_sqlite3_ok(sql());
1047 L(FL("prepared statement %s") % query.sql_cmd);
1048
1049 // no support for multiple statements here
1050 E(*tail == 0,
1051 F("multiple statements in query: %s") % query.sql_cmd);
1052 }
1053
1054 ncol = sqlite3_column_count(i->second.stmt());
1055
1056 E(want_cols == any_cols || want_cols == ncol,
1057 F("wanted %d columns got %d in query: %s") % want_cols % ncol % query.sql_cmd);
1058
1059 // bind parameters for this execution
1060
1061 int params = sqlite3_bind_parameter_count(i->second.stmt());
1062
1063 // Ensure that exactly the right number of parameters were given
1064 I(params == int(query.args.size()));
1065
1066 L(FL("binding %d parameters for %s") % params % query.sql_cmd);
1067
1068 for (int param = 1; param <= params; param++)
1069 {
1070 // profiling finds this logging to be quite expensive
1071 if (global_sanity.debug_p())
1072 {
1073 string prefix;
1074 string log(query.args[param-1].data);
1075
1076 if (query.args[param-1].type == query_param::blob)
1077 {
1078 prefix = "x";
1079 log = encode_hexenc(log);
1080 }
1081
1082 if (log.size() > constants::db_log_line_sz)
1083 log = log.substr(0, constants::db_log_line_sz - 2) + "..";
1084
1085 L(FL("binding %d with value '%s'") % param % log);
1086 }
1087
1088 switch (idx(query.args, param - 1).type)
1089 {
1090 case query_param::text:
1091 sqlite3_bind_text(i->second.stmt(), param,
1092 idx(query.args, param - 1).data.c_str(), -1,
1093 SQLITE_STATIC);
1094 break;
1095 case query_param::blob:
1096 {
1097 string const & data = idx(query.args, param - 1).data;
1098 sqlite3_bind_blob(i->second.stmt(), param,
1099 data.data(), data.size(),
1100 SQLITE_STATIC);
1101 }
1102 break;
1103 default:
1104 I(false);
1105 }
1106
1107 assert_sqlite3_ok(sql());
1108 }
1109
1110 // execute and process results
1111
1112 nrow = 0;
1113 for (rescode = sqlite3_step(i->second.stmt()); rescode == SQLITE_ROW;
1114 rescode = sqlite3_step(i->second.stmt()))
1115 {
1116 vector<string> row;
1117 for (int col = 0; col < ncol; col++)
1118 {
1119 const char * value = (const char*)sqlite3_column_blob(i->second.stmt(), col);
1120 int bytes = sqlite3_column_bytes(i->second.stmt(), col);
1121 E(value, F("null result in query: %s") % query.sql_cmd);
1122 row.push_back(string(value, value + bytes));
1123 //L(FL("row %d col %d value='%s'") % nrow % col % value);
1124 }
1125 res.push_back(row);
1126 }
1127
1128 if (rescode != SQLITE_DONE)
1129 assert_sqlite3_ok(sql());
1130
1131 sqlite3_reset(i->second.stmt());
1132 assert_sqlite3_ok(sql());
1133
1134 nrow = res.size();
1135
1136 i->second.count++;
1137
1138 E(want_rows == any_rows || want_rows == nrow,
1139 F("wanted %d rows got %d in query: %s") % want_rows % nrow % query.sql_cmd);
1140}
1141
1142bool
1143database_impl::table_has_entry(id const & key,
1144 std::string const & column,
1145 std::string const & table)
1146{
1147 results res;
1148 query q("SELECT 1 FROM " + table + " WHERE " + column + " = ? LIMIT 1");
1149 fetch(res, one_col, any_rows, q % blob(key()));
1150 return !res.empty();
1151}
1152
1153// general application-level logic
1154
1155void
1156database_impl::begin_transaction(bool exclusive)
1157{
1158 if (transaction_level == 0)
1159 {
1160 I(delayed_files.empty());
1161 I(roster_cache.all_clean());
1162 if (exclusive)
1163 execute(query("BEGIN EXCLUSIVE"));
1164 else
1165 execute(query("BEGIN DEFERRED"));
1166 transaction_exclusive = exclusive;
1167 }
1168 else
1169 {
1170 // You can't start an exclusive transaction within a non-exclusive
1171 // transaction
1172 I(!exclusive || transaction_exclusive);
1173 }
1174 transaction_level++;
1175}
1176
1177
1178static size_t
1179size_delayed_file(file_id const & id, file_data const & dat)
1180{
1181 return id.inner()().size() + dat.inner()().size();
1182}
1183
1184bool
1185database_impl::have_delayed_file(file_id const & id)
1186{
1187 return delayed_files.find(id) != delayed_files.end();
1188}
1189
1190void
1191database_impl::load_delayed_file(file_id const & id, file_data & dat)
1192{
1193 dat = safe_get(delayed_files, id);
1194}
1195
1196// precondition: have_delayed_file(an_id) == true
1197void
1198database_impl::cancel_delayed_file(file_id const & an_id)
1199{
1200 file_data const & dat = safe_get(delayed_files, an_id);
1201 size_t cancel_size = size_delayed_file(an_id, dat);
1202 I(cancel_size <= delayed_writes_size);
1203 delayed_writes_size -= cancel_size;
1204
1205 safe_erase(delayed_files, an_id);
1206}
1207
1208void
1209database_impl::drop_or_cancel_file(file_id const & id)
1210{
1211 if (have_delayed_file(id))
1212 cancel_delayed_file(id);
1213 else
1214 drop(id.inner(), "files");
1215}
1216
1217void
1218database_impl::schedule_delayed_file(file_id const & an_id,
1219 file_data const & dat)
1220{
1221 if (!have_delayed_file(an_id))
1222 {
1223 safe_insert(delayed_files, make_pair(an_id, dat));
1224 delayed_writes_size += size_delayed_file(an_id, dat);
1225 }
1226 if (delayed_writes_size > constants::db_max_delayed_file_bytes)
1227 flush_delayed_writes();
1228}
1229
1230void
1231database_impl::flush_delayed_writes()
1232{
1233 for (map<file_id, file_data>::const_iterator i = delayed_files.begin();
1234 i != delayed_files.end(); ++i)
1235 write_delayed_file(i->first, i->second);
1236 clear_delayed_writes();
1237}
1238
1239void
1240database_impl::clear_delayed_writes()
1241{
1242 delayed_files.clear();
1243 delayed_writes_size = 0;
1244}
1245
1246void
1247database_impl::roster_writeback_manager::writeout(revision_id const & id,
1248 cached_roster const & cr)
1249{
1250 I(cr.first);
1251 I(cr.second);
1252 imp.write_delayed_roster(id, *(cr.first), *(cr.second));
1253}
1254
1255void
1256database_impl::commit_transaction()
1257{
1258 if (transaction_level == 1)
1259 {
1260 flush_delayed_writes();
1261 roster_cache.clean_all();
1262 execute(query("COMMIT"));
1263 }
1264 transaction_level--;
1265}
1266
1267void
1268database_impl::rollback_transaction()
1269{
1270 if (transaction_level == 1)
1271 {
1272 clear_delayed_writes();
1273 roster_cache.clear_and_drop_writes();
1274 execute(query("ROLLBACK"));
1275 }
1276 transaction_level--;
1277}
1278
1279
1280bool
1281database_impl::file_or_manifest_base_exists(id const & ident,
1282 string const & table)
1283{
1284 // just check for a delayed file, since there are no delayed manifests
1285 if (have_delayed_file(file_id(ident)))
1286 return true;
1287 return table_has_entry(ident, "id", table);
1288}
1289
1290bool
1291database::file_or_manifest_base_exists(file_id const & ident,
1292 string const & table)
1293{
1294 return imp->file_or_manifest_base_exists(ident.inner(), table);
1295}
1296
1297// returns true if we are currently storing (or planning to store) a
1298// full-text for 'ident'
1299bool
1300database_impl::roster_base_stored(revision_id const & ident)
1301{
1302 if (roster_cache.exists(ident) && roster_cache.is_dirty(ident))
1303 return true;
1304 return table_has_entry(ident.inner(), "id", "rosters");
1305}
1306
1307// returns true if we currently have a full-text for 'ident' available
1308// (possibly cached). Warning: the results of this method are invalidated
1309// by calling roster_cache.insert_{clean,dirty}, because they can trigger
1310// cache cleaning.
1311bool
1312database_impl::roster_base_available(revision_id const & ident)
1313{
1314 if (roster_cache.exists(ident))
1315 return true;
1316 return table_has_entry(ident.inner(), "id", "rosters");
1317}
1318
1319bool
1320database::delta_exists(id const & ident,
1321 string const & table)
1322{
1323 return imp->table_has_entry(ident, "id", table);
1324}
1325
1326bool
1327database_impl::delta_exists(file_id const & ident,
1328 file_id const & base,
1329 string const & table)
1330{
1331 results res;
1332 query q("SELECT 1 FROM " + table + " WHERE id = ? and base = ? LIMIT 1");
1333 fetch(res, one_col, any_rows,
1334 q % blob(ident.inner()()) % blob(base.inner()()));
1335 return !res.empty();
1336}
1337
1338string
1339database_impl::count(string const & table)
1340{
1341 try
1342 {
1343 results res;
1344 query q("SELECT COUNT(*) FROM " + table);
1345 fetch(res, one_col, one_row, q);
1346 return (F("%u") % lexical_cast<u64>(res[0][0])).str();
1347 }
1348 catch (informative_failure const & e)
1349 {
1350 return format_sqlite_error_for_info(e);
1351 }
1352
1353}
1354
1355string
1356database_impl::space(string const & table, string const & rowspace, u64 & total)
1357{
1358 try
1359 {
1360 results res;
1361 // SUM({empty set}) is NULL; TOTAL({empty set}) is 0.0
1362 query q("SELECT TOTAL(" + rowspace + ") FROM " + table);
1363 fetch(res, one_col, one_row, q);
1364 u64 bytes = static_cast<u64>(lexical_cast<double>(res[0][0]));
1365 total += bytes;
1366 return (F("%u") % bytes).str();
1367 }
1368 catch (informative_failure & e)
1369 {
1370 return format_sqlite_error_for_info(e);
1371 }
1372}
1373
1374unsigned int
1375database_impl::page_size()
1376{
1377 results res;
1378 query q("PRAGMA page_size");
1379 fetch(res, one_col, one_row, q);
1380 return lexical_cast<unsigned int>(res[0][0]);
1381}
1382
1383unsigned int
1384database_impl::cache_size()
1385{
1386 // This returns the persistent (default) cache size. It's possible to
1387 // override this setting transiently at runtime by setting PRAGMA
1388 // cache_size.
1389 results res;
1390 query q("PRAGMA default_cache_size");
1391 fetch(res, one_col, one_row, q);
1392 return lexical_cast<unsigned int>(res[0][0]);
1393}
1394
1395void
1396database_impl::get_ids(string const & table, set<id> & ids)
1397{
1398 results res;
1399 query q("SELECT id FROM " + table);
1400 fetch(res, one_col, any_rows, q);
1401
1402 for (size_t i = 0; i < res.size(); ++i)
1403 {
1404 ids.insert(id(res[i][0]));
1405 }
1406}
1407
1408// for files and legacy manifest support
1409void
1410database_impl::get_file_or_manifest_base_unchecked(id const & ident,
1411 data & dat,
1412 string const & table)
1413{
1414 if (have_delayed_file(file_id(ident)))
1415 {
1416 file_data tmp;
1417 load_delayed_file(file_id(ident), tmp);
1418 dat = tmp.inner();
1419 return;
1420 }
1421
1422 results res;
1423 query q("SELECT data FROM " + table + " WHERE id = ?");
1424 fetch(res, one_col, one_row, q % blob(ident()));
1425
1426 gzip<data> rdata(res[0][0]);
1427 data rdata_unpacked;
1428 decode_gzip(rdata,rdata_unpacked);
1429
1430 dat = rdata_unpacked;
1431}
1432
1433// for files and legacy manifest support
1434void
1435database_impl::get_file_or_manifest_delta_unchecked(id const & ident,
1436 id const & base,
1437 delta & del,
1438 string const & table)
1439{
1440 I(ident() != "");
1441 I(base() != "");
1442 results res;
1443 query q("SELECT delta FROM " + table + " WHERE id = ? AND base = ?");
1444 fetch(res, one_col, one_row,
1445 q % blob(ident()) % blob(base()));
1446
1447 gzip<delta> del_packed(res[0][0]);
1448 decode_gzip(del_packed, del);
1449}
1450
1451void
1452database_impl::get_roster_base(revision_id const & ident,
1453 roster_t & roster, marking_map & marking)
1454{
1455 if (roster_cache.exists(ident))
1456 {
1457 cached_roster cr;
1458 roster_cache.fetch(ident, cr);
1459 I(cr.first);
1460 roster = *(cr.first);
1461 I(cr.second);
1462 marking = *(cr.second);
1463 return;
1464 }
1465 results res;
1466 query q("SELECT checksum, data FROM rosters WHERE id = ?");
1467 fetch(res, 2, one_row, q % blob(ident.inner()()));
1468
1469 id checksum(res[0][0]);
1470 id calculated;
1471 calculate_ident(data(res[0][1]), calculated);
1472 I(calculated == checksum);
1473
1474 gzip<data> dat_packed(res[0][1]);
1475 data dat;
1476 decode_gzip(dat_packed, dat);
1477 read_roster_and_marking(roster_data(dat), roster, marking);
1478}
1479
1480void
1481database_impl::get_roster_delta(id const & ident,
1482 id const & base,
1483 roster<delta> & del)
1484{
1485 results res;
1486 query q("SELECT checksum, delta FROM roster_deltas WHERE id = ? AND base = ?");
1487 fetch(res, 2, one_row, q % blob(ident()) % blob(base()));
1488
1489 id checksum(res[0][0]);
1490 id calculated;
1491 calculate_ident(data(res[0][1]), calculated);
1492 I(calculated == checksum);
1493
1494 gzip<delta> del_packed(res[0][1]);
1495 delta tmp;
1496 decode_gzip(del_packed, tmp);
1497 del = roster<delta>(tmp);
1498}
1499
1500void
1501database_impl::write_delayed_file(file_id const & ident,
1502 file_data const & dat)
1503{
1504 gzip<data> dat_packed;
1505 encode_gzip(dat.inner(), dat_packed);
1506
1507 // ident is a hash, which we should check
1508 I(!null_id(ident));
1509 file_id tid;
1510 calculate_ident(dat, tid);
1511 MM(ident);
1512 MM(tid);
1513 I(tid == ident);
1514 // and then write things to the db
1515 query q("INSERT INTO files (id, data) VALUES (?, ?)");
1516 execute(q % blob(ident.inner()()) % blob(dat_packed()));
1517}
1518
1519void
1520database_impl::write_delayed_roster(revision_id const & ident,
1521 roster_t const & roster,
1522 marking_map const & marking)
1523{
1524 roster_data dat;
1525 write_roster_and_marking(roster, marking, dat);
1526 gzip<data> dat_packed;
1527 encode_gzip(dat.inner(), dat_packed);
1528
1529 // ident is a number, and we should calculate a checksum on what
1530 // we write
1531 id checksum;
1532 calculate_ident(data(dat_packed()), checksum);
1533
1534 // and then write it
1535 execute(query("INSERT INTO rosters (id, checksum, data) VALUES (?, ?, ?)")
1536 % blob(ident.inner()())
1537 % blob(checksum())
1538 % blob(dat_packed()));
1539}
1540
1541
1542void
1543database::put_file_delta(file_id const & ident,
1544 file_id const & base,
1545 file_delta const & del)
1546{
1547 // nb: delta schema is (id, base, delta)
1548 I(!null_id(ident));
1549 I(!null_id(base));
1550
1551 gzip<delta> del_packed;
1552 encode_gzip(del.inner(), del_packed);
1553
1554 imp->execute(query("INSERT INTO file_deltas VALUES (?, ?, ?)")
1555 % blob(ident.inner()())
1556 % blob(base.inner()())
1557 % blob(del_packed()));
1558}
1559
1560void
1561database_impl::put_roster_delta(revision_id const & ident,
1562 revision_id const & base,
1563 roster_delta const & del)
1564{
1565 gzip<delta> del_packed;
1566 encode_gzip(del.inner(), del_packed);
1567
1568 id checksum;
1569 calculate_ident(data(del_packed()), checksum);
1570
1571 query q("INSERT INTO roster_deltas (id, base, checksum, delta) VALUES (?, ?, ?, ?)");
1572 execute(q
1573 % blob(ident.inner()())
1574 % blob(base.inner()())
1575 % blob(checksum())
1576 % blob(del_packed()));
1577}
1578
1579struct file_and_manifest_reconstruction_graph : public reconstruction_graph
1580{
1581 database_impl & imp;
1582 string const & data_table;
1583 string const & delta_table;
1584
1585 file_and_manifest_reconstruction_graph(database_impl & imp,
1586 string const & data_table,
1587 string const & delta_table)
1588 : imp(imp), data_table(data_table), delta_table(delta_table)
1589 {}
1590 virtual bool is_base(id const & node) const
1591 {
1592 return imp.vcache.exists(node)
1593 || imp.file_or_manifest_base_exists(node, data_table);
1594 }
1595 virtual void get_next(id const & from, set<id> & next) const
1596 {
1597 next.clear();
1598 results res;
1599 query q("SELECT base FROM " + delta_table + " WHERE id = ?");
1600 imp.fetch(res, one_col, any_rows, q % blob(from()));
1601 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
1602 next.insert(id((*i)[0]));
1603 }
1604};
1605
1606// used for files and legacy manifest migration
1607void
1608database_impl::get_version(id const & ident,
1609 data & dat,
1610 string const & data_table,
1611 string const & delta_table)
1612{
1613 I(ident() != "");
1614
1615 reconstruction_path selected_path;
1616 {
1617 file_and_manifest_reconstruction_graph graph(*this, data_table, delta_table);
1618 get_reconstruction_path(ident, graph, selected_path);
1619 }
1620
1621 I(!selected_path.empty());
1622
1623 id curr = selected_path.back();
1624 selected_path.pop_back();
1625 data begin;
1626
1627 if (vcache.exists(curr))
1628 I(vcache.fetch(curr, begin));
1629 else
1630 get_file_or_manifest_base_unchecked(curr, begin, data_table);
1631
1632 shared_ptr<delta_applicator> appl = new_piecewise_applicator();
1633 appl->begin(begin());
1634
1635 for (reconstruction_path::reverse_iterator i = selected_path.rbegin();
1636 i != selected_path.rend(); ++i)
1637 {
1638 id const nxt = id(*i);
1639
1640 if (!vcache.exists(curr))
1641 {
1642 string tmp;
1643 appl->finish(tmp);
1644 vcache.insert_clean(curr, data(tmp));
1645 }
1646
1647 if (global_sanity.debug_p())
1648 L(FL("following delta %s -> %s") % curr % nxt);
1649 delta del;
1650 get_file_or_manifest_delta_unchecked(nxt, curr, del, delta_table);
1651 apply_delta(appl, del());
1652
1653 appl->next();
1654 curr = nxt;
1655 }
1656
1657 string tmp;
1658 appl->finish(tmp);
1659 dat = data(tmp);
1660
1661 id final;
1662 calculate_ident(dat, final);
1663 I(final == ident);
1664
1665 if (!vcache.exists(ident))
1666 vcache.insert_clean(ident, dat);
1667}
1668
1669struct roster_reconstruction_graph : public reconstruction_graph
1670{
1671 database_impl & imp;
1672 roster_reconstruction_graph(database_impl & imp) : imp(imp) {}
1673 virtual bool is_base(id const & node) const
1674 {
1675 return imp.roster_base_available(revision_id(node));
1676 }
1677 virtual void get_next(id const & from, set<id> & next) const
1678 {
1679 next.clear();
1680 results res;
1681 query q("SELECT base FROM roster_deltas WHERE id = ?");
1682 imp.fetch(res, one_col, any_rows, q % blob(from()));
1683 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
1684 next.insert(id((*i)[0]));
1685 }
1686};
1687
1688struct database_impl::extractor
1689{
1690 virtual bool look_at_delta(roster_delta const & del) = 0;
1691 virtual void look_at_roster(roster_t const & roster, marking_map const & mm) = 0;
1692 virtual ~extractor() {};
1693};
1694
1695struct database_impl::markings_extractor : public database_impl::extractor
1696{
1697private:
1698 node_id const & nid;
1699 marking_t & markings;
1700
1701public:
1702 markings_extractor(node_id const & _nid, marking_t & _markings) :
1703 nid(_nid), markings(_markings) {} ;
1704
1705 bool look_at_delta(roster_delta const & del)
1706 {
1707 return try_get_markings_from_roster_delta(del, nid, markings);
1708 }
1709
1710 void look_at_roster(roster_t const & roster, marking_map const & mm)
1711 {
1712 marking_map::const_iterator mmi =
1713 mm.find(nid);
1714 I(mmi != mm.end());
1715 markings = mmi->second;
1716 }
1717};
1718
1719struct database_impl::file_content_extractor : database_impl::extractor
1720{
1721private:
1722 node_id const & nid;
1723 file_id & content;
1724
1725public:
1726 file_content_extractor(node_id const & _nid, file_id & _content) :
1727 nid(_nid), content(_content) {} ;
1728
1729 bool look_at_delta(roster_delta const & del)
1730 {
1731 return try_get_content_from_roster_delta(del, nid, content);
1732 }
1733
1734 void look_at_roster(roster_t const & roster, marking_map const & mm)
1735 {
1736 if (roster.has_node(nid))
1737 content = downcast_to_file_t(roster.get_node(nid))->content;
1738 else
1739 content = file_id();
1740 }
1741};
1742
1743void
1744database_impl::extract_from_deltas(revision_id const & ident, extractor & x)
1745{
1746 reconstruction_path selected_path;
1747 {
1748 roster_reconstruction_graph graph(*this);
1749 {
1750 // we look at the nearest delta(s) first, without constructing the
1751 // whole path, as that would be a rather expensive operation.
1752 //
1753 // the reason why this strategy is worth the effort is, that in most
1754 // cases we are looking at the parent of a (content-)marked node, thus
1755 // the information we are for is right there in the delta leading to
1756 // this node.
1757 //
1758 // recording the deltas visited here in a set as to avoid inspecting
1759 // them later seems to be of little value, as it imposes a cost here,
1760 // but can seldom be exploited.
1761 set<id> deltas;
1762 graph.get_next(ident.inner(), deltas);
1763 for (set<id>::const_iterator i = deltas.begin();
1764 i != deltas.end(); ++i)
1765 {
1766 roster_delta del;
1767 get_roster_delta(ident.inner(), *i, del);
1768 bool found = x.look_at_delta(del);
1769 if (found)
1770 return;
1771 }
1772 }
1773 get_reconstruction_path(ident.inner(), graph, selected_path);
1774 }
1775
1776 int path_length(selected_path.size());
1777 int i(0);
1778 id target_rev;
1779
1780 for (reconstruction_path::const_iterator p = selected_path.begin();
1781 p != selected_path.end(); ++p)
1782 {
1783 if (i > 0)
1784 {
1785 roster_delta del;
1786 get_roster_delta(target_rev, id(*p), del);
1787 bool found = x.look_at_delta(del);
1788 if (found)
1789 return;
1790 }
1791 if (i == path_length-1)
1792 {
1793 // last iteration, we have reached a roster base
1794 roster_t roster;
1795 marking_map mm;
1796 get_roster_base(revision_id(*p), roster, mm);
1797 x.look_at_roster(roster, mm);
1798 return;
1799 }
1800 target_rev = id(*p);
1801 ++i;
1802 }
1803}
1804
1805void
1806database::get_markings(revision_id const & id,
1807 node_id const & nid,
1808 marking_t & markings)
1809{
1810 database_impl::markings_extractor x(nid, markings);
1811 imp->extract_from_deltas(id, x);
1812}
1813
1814void
1815database::get_file_content(revision_id const & id,
1816 node_id const & nid,
1817 file_id & content)
1818{
1819 // the imaginary root revision doesn't have any file.
1820 if (null_id(id))
1821 {
1822 content = file_id();
1823 return;
1824 }
1825 database_impl::file_content_extractor x(nid, content);
1826 imp->extract_from_deltas(id, x);
1827}
1828
1829void
1830database::get_roster_version(revision_id const & ros_id,
1831 cached_roster & cr)
1832{
1833 // if we already have it, exit early
1834 if (imp->roster_cache.exists(ros_id))
1835 {
1836 imp->roster_cache.fetch(ros_id, cr);
1837 return;
1838 }
1839
1840 reconstruction_path selected_path;
1841 {
1842 roster_reconstruction_graph graph(*imp);
1843 get_reconstruction_path(ros_id.inner(), graph, selected_path);
1844 }
1845
1846 id curr(selected_path.back());
1847 selected_path.pop_back();
1848 // we know that this isn't already in the cache (because of the early exit
1849 // above), so we should create new objects and spend time filling them in.
1850 shared_ptr<roster_t> roster(new roster_t);
1851 shared_ptr<marking_map> marking(new marking_map);
1852 imp->get_roster_base(revision_id(curr), *roster, *marking);
1853
1854 for (reconstruction_path::reverse_iterator i = selected_path.rbegin();
1855 i != selected_path.rend(); ++i)
1856 {
1857 id const nxt(*i);
1858 if (global_sanity.debug_p())
1859 L(FL("following delta %s -> %s") % curr % nxt);
1860 roster_delta del;
1861 imp->get_roster_delta(nxt, curr, del);
1862 apply_roster_delta(del, *roster, *marking);
1863 curr = nxt;
1864 }
1865
1866 // Double-check that the thing we got out looks okay. We know that when
1867 // the roster was written to the database, it passed both of these tests,
1868 // and we also know that the data on disk has passed our checks for data
1869 // corruption -- so in theory, we know that what we got out is exactly
1870 // what we put in, and these checks are redundant. (They cannot catch all
1871 // possible errors in any case, e.g., they don't test that the marking is
1872 // correct.) What they can do, though, is serve as a sanity check on the
1873 // delta reconstruction code; if there is a bug where we put something
1874 // into the database and then later get something different back out, then
1875 // this is the only thing that can catch it.
1876 roster->check_sane_against(*marking);
1877 manifest_id expected_mid, actual_mid;
1878 get_revision_manifest(ros_id, expected_mid);
1879 calculate_ident(*roster, actual_mid);
1880 I(expected_mid == actual_mid);
1881
1882 // const'ify the objects, to save them and pass them out
1883 cr.first = roster;
1884 cr.second = marking;
1885 imp->roster_cache.insert_clean(ros_id, cr);
1886}
1887
1888
1889void
1890database_impl::drop(id const & ident,
1891 string const & table)
1892{
1893 string drop = "DELETE FROM " + table + " WHERE id = ?";
1894 execute(query(drop) % blob(ident()));
1895}
1896
1897// ------------------------------------------------------------
1898// -- --
1899// -- public interface follows --
1900// -- --
1901// ------------------------------------------------------------
1902
1903bool
1904database::file_version_exists(file_id const & id)
1905{
1906 return delta_exists(id.inner(), "file_deltas")
1907 || imp->file_or_manifest_base_exists(id.inner(), "files");
1908}
1909
1910bool
1911database::roster_version_exists(revision_id const & id)
1912{
1913 return delta_exists(id.inner(), "roster_deltas")
1914 || imp->roster_base_available(id);
1915}
1916
1917bool
1918database::revision_exists(revision_id const & id)
1919{
1920 results res;
1921 query q("SELECT id FROM revisions WHERE id = ?");
1922 imp->fetch(res, one_col, any_rows, q % blob(id.inner()()));
1923 I(res.size() <= 1);
1924 return res.size() == 1;
1925}
1926
1927void
1928database::get_file_ids(set<file_id> & ids)
1929{
1930 ids.clear();
1931 set<id> tmp;
1932 imp->get_ids("files", tmp);
1933 imp->get_ids("file_deltas", tmp);
1934 add_decoration_to_container(tmp, ids);
1935}
1936
1937void
1938database::get_revision_ids(set<revision_id> & ids)
1939{
1940 ids.clear();
1941 set<id> tmp;
1942 imp->get_ids("revisions", tmp);
1943 add_decoration_to_container(tmp, ids);
1944}
1945
1946void
1947database::get_roster_ids(set<revision_id> & ids)
1948{
1949 ids.clear();
1950 set<id> tmp;
1951 imp->get_ids("rosters", tmp);
1952 add_decoration_to_container(tmp, ids);
1953 imp->get_ids("roster_deltas", tmp);
1954 add_decoration_to_container(tmp, ids);
1955}
1956
1957void
1958database::get_file_version(file_id const & id,
1959 file_data & dat)
1960{
1961 data tmp;
1962 imp->get_version(id.inner(), tmp, "files", "file_deltas");
1963 dat = file_data(tmp);
1964}
1965
1966void
1967database::get_manifest_version(manifest_id const & id,
1968 manifest_data & dat)
1969{
1970 data tmp;
1971 imp->get_version(id.inner(), tmp, "manifests", "manifest_deltas");
1972 dat = manifest_data(tmp);
1973}
1974
1975void
1976database::put_file(file_id const & id,
1977 file_data const & dat)
1978{
1979 if (file_version_exists(id))
1980 L(FL("file version '%s' already exists in db") % id);
1981 else
1982 imp->schedule_delayed_file(id, dat);
1983}
1984
1985void
1986database::put_file_version(file_id const & old_id,
1987 file_id const & new_id,
1988 file_delta const & del)
1989{
1990 I(!(old_id == new_id));
1991 file_data old_data, new_data;
1992 file_delta reverse_delta;
1993
1994 if (!file_version_exists(old_id))
1995 {
1996 W(F("file preimage '%s' missing in db") % old_id);
1997 W(F("dropping delta '%s' -> '%s'") % old_id % new_id);
1998 return;
1999 }
2000
2001 get_file_version(old_id, old_data);
2002 {
2003 data tmp;
2004 patch(old_data.inner(), del.inner(), tmp);
2005 new_data = file_data(tmp);
2006 }
2007
2008 {
2009 string tmp;
2010 invert_xdelta(old_data.inner()(), del.inner()(), tmp);
2011 reverse_delta = file_delta(tmp);
2012 data old_tmp;
2013 patch(new_data.inner(), reverse_delta.inner(), old_tmp);
2014 // We already have the real old data, so compare the
2015 // reconstruction to that directly instead of hashing
2016 // the reconstruction and comparing hashes.
2017 I(old_tmp == old_data.inner());
2018 }
2019
2020 transaction_guard guard(*this);
2021 if (file_or_manifest_base_exists(old_id, "files"))
2022 {
2023 // descendent of a head version replaces the head, therefore old head
2024 // must be disposed of
2025 imp->drop_or_cancel_file(old_id);
2026 }
2027 if (!file_or_manifest_base_exists(new_id, "files"))
2028 {
2029 imp->schedule_delayed_file(new_id, new_data);
2030 imp->drop(new_id.inner(), "file_deltas");
2031 }
2032
2033 if (!imp->delta_exists(old_id, new_id, "file_deltas"))
2034 {
2035 put_file_delta(old_id, new_id, reverse_delta);
2036 guard.commit();
2037 }
2038}
2039
2040void
2041database::get_arbitrary_file_delta(file_id const & src_id,
2042 file_id const & dst_id,
2043 file_delta & del)
2044{
2045 delta dtmp;
2046 // Deltas stored in the database go from base -> id.
2047 results res;
2048 query q1("SELECT delta FROM file_deltas "
2049 "WHERE base = ? AND id = ?");
2050 imp->fetch(res, one_col, any_rows,
2051 q1 % blob(src_id.inner()()) % blob(dst_id.inner()()));
2052
2053 if (!res.empty())
2054 {
2055 // Exact hit: a plain delta from src -> dst.
2056 gzip<delta> del_packed(res[0][0]);
2057 decode_gzip(del_packed, dtmp);
2058 del = file_delta(dtmp);
2059 return;
2060 }
2061
2062 query q2("SELECT delta FROM file_deltas "
2063 "WHERE base = ? AND id = ?");
2064 imp->fetch(res, one_col, any_rows,
2065 q2 % blob(dst_id.inner()()) % blob(src_id.inner()()));
2066
2067 if (!res.empty())
2068 {
2069 // We have a delta from dst -> src; we need to
2070 // invert this to a delta from src -> dst.
2071 gzip<delta> del_packed(res[0][0]);
2072 decode_gzip(del_packed, dtmp);
2073 string fwd_delta;
2074 file_data dst;
2075 get_file_version(dst_id, dst);
2076 invert_xdelta(dst.inner()(), dtmp(), fwd_delta);
2077 del = file_delta(fwd_delta);
2078 return;
2079 }
2080
2081 // No deltas of use; just load both versions and diff.
2082 file_data fd1, fd2;
2083 get_file_version(src_id, fd1);
2084 get_file_version(dst_id, fd2);
2085 diff(fd1.inner(), fd2.inner(), dtmp);
2086 del = file_delta(dtmp);
2087}
2088
2089
2090void
2091database::get_revision_ancestry(rev_ancestry_map & graph)
2092{
2093 // share some storage
2094 id::symtab id_syms;
2095
2096 results res;
2097 graph.clear();
2098 imp->fetch(res, 2, any_rows,
2099 query("SELECT parent,child FROM revision_ancestry"));
2100 for (size_t i = 0; i < res.size(); ++i)
2101 graph.insert(make_pair(revision_id(res[i][0]),
2102 revision_id(res[i][1])));
2103}
2104
2105void
2106database::get_revision_parents(revision_id const & id,
2107 set<revision_id> & parents)
2108{
2109 I(!null_id(id));
2110 parent_id_map::iterator i = imp->parent_cache.find(id);
2111 if (i == imp->parent_cache.end())
2112 {
2113 results res;
2114 parents.clear();
2115 imp->fetch(res, one_col, any_rows,
2116 query("SELECT parent FROM revision_ancestry WHERE child = ?")
2117 % blob(id.inner()()));
2118 for (size_t i = 0; i < res.size(); ++i)
2119 parents.insert(revision_id(res[i][0]));
2120
2121 imp->parent_cache.insert(make_pair(id, parents));
2122 }
2123 else
2124 {
2125 parents = i->second;
2126 }
2127}
2128
2129void
2130database::get_revision_children(revision_id const & id,
2131 set<revision_id> & children)
2132{
2133 results res;
2134 children.clear();
2135 imp->fetch(res, one_col, any_rows,
2136 query("SELECT child FROM revision_ancestry WHERE parent = ?")
2137 % blob(id.inner()()));
2138 for (size_t i = 0; i < res.size(); ++i)
2139 children.insert(revision_id(res[i][0]));
2140}
2141
2142void
2143database::get_leaves(set<revision_id> & leaves)
2144{
2145 results res;
2146 leaves.clear();
2147 imp->fetch(res, one_col, any_rows,
2148 query("SELECT revisions.id FROM revisions "
2149 "LEFT JOIN revision_ancestry "
2150 "ON revisions.id = revision_ancestry.parent "
2151 "WHERE revision_ancestry.child IS null"));
2152 for (size_t i = 0; i < res.size(); ++i)
2153 leaves.insert(revision_id(res[i][0]));
2154}
2155
2156
2157void
2158database::get_revision_manifest(revision_id const & rid,
2159 manifest_id & mid)
2160{
2161 revision_t rev;
2162 get_revision(rid, rev);
2163 mid = rev.new_manifest;
2164}
2165
2166void
2167database::get_common_ancestors(std::set<revision_id> const & revs,
2168 std::set<revision_id> & common_ancestors)
2169{
2170 set<revision_id> ancestors, all_common_ancestors;
2171 vector<revision_id> frontier;
2172 for (set<revision_id>::const_iterator i = revs.begin();
2173 i != revs.end(); ++i)
2174 {
2175 I(revision_exists(*i));
2176 ancestors.clear();
2177 ancestors.insert(*i);
2178 frontier.push_back(*i);
2179 while (!frontier.empty())
2180 {
2181 revision_id rid = frontier.back();
2182 frontier.pop_back();
2183 if(!null_id(rid))
2184 {
2185 set<revision_id> parents;
2186 get_revision_parents(rid, parents);
2187 for (set<revision_id>::const_iterator i = parents.begin();
2188 i != parents.end(); ++i)
2189 {
2190 if (ancestors.find(*i) == ancestors.end())
2191 {
2192 frontier.push_back(*i);
2193 ancestors.insert(*i);
2194 }
2195 }
2196 }
2197 }
2198 if (all_common_ancestors.empty())
2199 all_common_ancestors = ancestors;
2200 else
2201 {
2202 set<revision_id> common;
2203 set_intersection(ancestors.begin(), ancestors.end(),
2204 all_common_ancestors.begin(), all_common_ancestors.end(),
2205 inserter(common, common.begin()));
2206 all_common_ancestors = common;
2207 }
2208 }
2209
2210 for (set<revision_id>::const_iterator i = all_common_ancestors.begin();
2211 i != all_common_ancestors.end(); ++i)
2212 {
2213 // FIXME: where do these null'ed IDs come from?
2214 if (null_id(*i)) continue;
2215 common_ancestors.insert(*i);
2216 }
2217}
2218
2219void
2220database::get_revision(revision_id const & id,
2221 revision_t & rev)
2222{
2223 revision_data d;
2224 get_revision(id, d);
2225 read_revision(d, rev);
2226}
2227
2228void
2229database::get_revision(revision_id const & id,
2230 revision_data & dat)
2231{
2232 I(!null_id(id));
2233 results res;
2234 imp->fetch(res, one_col, one_row,
2235 query("SELECT data FROM revisions WHERE id = ?")
2236 % blob(id.inner()()));
2237
2238 gzip<data> gzdata(res[0][0]);
2239 data rdat;
2240 decode_gzip(gzdata,rdat);
2241
2242 // verify that we got a revision with the right id
2243 {
2244 revision_id tmp;
2245 calculate_ident(revision_data(rdat), tmp);
2246 I(id == tmp);
2247 }
2248
2249 dat = revision_data(rdat);
2250}
2251
2252void
2253database::get_rev_height(revision_id const & id,
2254 rev_height & height)
2255{
2256 if (null_id(id))
2257 {
2258 height = rev_height::root_height();
2259 return;
2260 }
2261
2262 height_map::const_iterator i = imp->height_cache.find(id);
2263 if (i == imp->height_cache.end())
2264 {
2265 results res;
2266 imp->fetch(res, one_col, one_row,
2267 query("SELECT height FROM heights WHERE revision = ?")
2268 % blob(id.inner()()));
2269
2270 I(res.size() == 1);
2271
2272 height = rev_height(res[0][0]);
2273 imp->height_cache.insert(make_pair(id, height));
2274 }
2275 else
2276 {
2277 height = i->second;
2278 }
2279
2280 I(height.valid());
2281}
2282
2283void
2284database::put_rev_height(revision_id const & id,
2285 rev_height const & height)
2286{
2287 I(!null_id(id));
2288 I(revision_exists(id));
2289 I(height.valid());
2290
2291 imp->height_cache.erase(id);
2292
2293 imp->execute(query("INSERT INTO heights VALUES(?, ?)")
2294 % blob(id.inner()())
2295 % blob(height()));
2296}
2297
2298bool
2299database::has_rev_height(rev_height const & height)
2300{
2301 results res;
2302 imp->fetch(res, one_col, any_rows,
2303 query("SELECT height FROM heights WHERE height = ?")
2304 % blob(height()));
2305 I((res.size() == 1) || (res.empty()));
2306 return res.size() == 1;
2307}
2308
2309void
2310database::deltify_revision(revision_id const & rid)
2311{
2312 transaction_guard guard(*this);
2313 revision_t rev;
2314 MM(rev);
2315 MM(rid);
2316 get_revision(rid, rev);
2317 // Make sure that all parent revs have their files replaced with deltas
2318 // from this rev's files.
2319 {
2320 for (edge_map::const_iterator i = rev.edges.begin();
2321 i != rev.edges.end(); ++i)
2322 {
2323 for (map<file_path, pair<file_id, file_id> >::const_iterator
2324 j = edge_changes(i).deltas_applied.begin();
2325 j != edge_changes(i).deltas_applied.end(); ++j)
2326 {
2327 if (file_or_manifest_base_exists(delta_entry_src(j), "files") &&
2328 file_version_exists(delta_entry_dst(j)))
2329 {
2330 file_data old_data;
2331 file_data new_data;
2332 get_file_version(delta_entry_src(j), old_data);
2333 get_file_version(delta_entry_dst(j), new_data);
2334 delta delt;
2335 diff(old_data.inner(), new_data.inner(), delt);
2336 file_delta del(delt);
2337 imp->drop_or_cancel_file(delta_entry_dst(j));
2338 imp->drop(delta_entry_dst(j).inner(), "file_deltas");
2339 put_file_version(delta_entry_src(j), delta_entry_dst(j), del);
2340 }
2341 }
2342 }
2343 }
2344 guard.commit();
2345}
2346
2347
2348bool
2349database::put_revision(revision_id const & new_id,
2350 revision_t const & rev)
2351{
2352 MM(new_id);
2353 MM(rev);
2354
2355 I(!null_id(new_id));
2356
2357 if (revision_exists(new_id))
2358 {
2359 if (global_sanity.debug_p())
2360 L(FL("revision '%s' already exists in db") % new_id);
2361 return false;
2362 }
2363
2364 I(rev.made_for == made_for_database);
2365 rev.check_sane();
2366
2367 // Phase 1: confirm the revision makes sense, and the required files
2368 // actually exist
2369 for (edge_map::const_iterator i = rev.edges.begin();
2370 i != rev.edges.end(); ++i)
2371 {
2372 if (!edge_old_revision(i).inner()().empty()
2373 && !revision_exists(edge_old_revision(i)))
2374 {
2375 W(F("missing prerequisite revision '%s'")
2376 % edge_old_revision(i));
2377 W(F("dropping revision '%s'") % new_id);
2378 return false;
2379 }
2380
2381 for (map<file_path, file_id>::const_iterator a
2382 = edge_changes(i).files_added.begin();
2383 a != edge_changes(i).files_added.end(); ++a)
2384 {
2385 if (! file_version_exists(a->second))
2386 {
2387 W(F("missing prerequisite file '%s'") % a->second);
2388 W(F("dropping revision '%s'") % new_id);
2389 return false;
2390 }
2391 }
2392
2393 for (map<file_path, pair<file_id, file_id> >::const_iterator d
2394 = edge_changes(i).deltas_applied.begin();
2395 d != edge_changes(i).deltas_applied.end(); ++d)
2396 {
2397 I(!delta_entry_src(d).inner()().empty());
2398 I(!delta_entry_dst(d).inner()().empty());
2399
2400 if (! file_version_exists(delta_entry_src(d)))
2401 {
2402 W(F("missing prerequisite file pre-delta '%s'")
2403 % delta_entry_src(d));
2404 W(F("dropping revision '%s'") % new_id);
2405 return false;
2406 }
2407
2408 if (! file_version_exists(delta_entry_dst(d)))
2409 {
2410 W(F("missing prerequisite file post-delta '%s'")
2411 % delta_entry_dst(d));
2412 W(F("dropping revision '%s'") % new_id);
2413 return false;
2414 }
2415 }
2416 }
2417
2418 transaction_guard guard(*this);
2419
2420 // Phase 2: Write the revision data (inside a transaction)
2421
2422 revision_data d;
2423 write_revision(rev, d);
2424 gzip<data> d_packed;
2425 encode_gzip(d.inner(), d_packed);
2426 imp->execute(query("INSERT INTO revisions VALUES(?, ?)")
2427 % blob(new_id.inner()())
2428 % blob(d_packed()));
2429
2430 for (edge_map::const_iterator e = rev.edges.begin();
2431 e != rev.edges.end(); ++e)
2432 {
2433 imp->execute(query("INSERT INTO revision_ancestry VALUES(?, ?)")
2434 % blob(edge_old_revision(e).inner()())
2435 % blob(new_id.inner()()));
2436 }
2437 // We don't have to clear out the child's entry in the parent_cache,
2438 // because the child did not exist before this function was called, so
2439 // it can't be in the parent_cache already.
2440
2441 // Phase 3: Construct and write the roster (which also checks the manifest
2442 // id as it goes), but only if the roster does not already exist in the db
2443 // (i.e. because it was left over by a kill_rev_locally)
2444 // FIXME: there is no knowledge yet on speed implications for commands which
2445 // put a lot of revisions in a row (i.e. tailor or cvs_import)!
2446
2447 if (!roster_version_exists(new_id))
2448 {
2449 put_roster_for_revision(new_id, rev);
2450 }
2451 else
2452 {
2453 L(FL("roster for revision '%s' already exists in db") % new_id);
2454 }
2455
2456 // Phase 4: rewrite any files that need deltas added
2457
2458 deltify_revision(new_id);
2459
2460 // Phase 5: determine the revision height
2461
2462 put_height_for_revision(new_id, rev);
2463
2464 // Finally, commit.
2465
2466 guard.commit();
2467 return true;
2468}
2469
2470void
2471database::put_height_for_revision(revision_id const & new_id,
2472 revision_t const & rev)
2473{
2474 I(!null_id(new_id));
2475
2476 rev_height highest_parent;
2477 // we always branch off the highest parent ...
2478 for (edge_map::const_iterator e = rev.edges.begin();
2479 e != rev.edges.end(); ++e)
2480 {
2481 rev_height parent; MM(parent);
2482 get_rev_height(edge_old_revision(e), parent);
2483 if (parent > highest_parent)
2484 {
2485 highest_parent = parent;
2486 }
2487 }
2488
2489 // ... then find the first unused child
2490 u32 childnr(0);
2491 rev_height candidate; MM(candidate);
2492 while(true)
2493 {
2494 candidate = highest_parent.child_height(childnr);
2495 if (!has_rev_height(candidate))
2496 {
2497 break;
2498 }
2499 I(childnr < std::numeric_limits<u32>::max());
2500 ++childnr;
2501 }
2502 put_rev_height(new_id, candidate);
2503}
2504
2505void
2506database::put_roster_for_revision(revision_id const & new_id,
2507 revision_t const & rev)
2508{
2509 // Construct, the roster, sanity-check the manifest id, and then write it
2510 // to the db
2511 shared_ptr<roster_t> ros_writeable(new roster_t); MM(*ros_writeable);
2512 shared_ptr<marking_map> mm_writeable(new marking_map); MM(*mm_writeable);
2513 manifest_id roster_manifest_id;
2514 MM(roster_manifest_id);
2515 make_roster_for_revision(*this, rev, new_id, *ros_writeable, *mm_writeable);
2516 calculate_ident(*ros_writeable, roster_manifest_id);
2517 made_from_t made_from(rev.made_from);
2518 I(rev.new_manifest == roster_manifest_id);
2519 // const'ify the objects, suitable for caching etc.
2520 roster_t_cp ros = ros_writeable;
2521 marking_map_cp mm = mm_writeable;
2522 put_roster(new_id, ros, mm);
2523}
2524
2525bool
2526database::put_revision(revision_id const & new_id,
2527 revision_data const & dat)
2528{
2529 revision_t rev;
2530 read_revision(dat, rev);
2531 return put_revision(new_id, rev);
2532}
2533
2534
2535void
2536database::delete_existing_revs_and_certs()
2537{
2538 imp->execute(query("DELETE FROM revisions"));
2539 imp->execute(query("DELETE FROM revision_ancestry"));
2540 imp->execute(query("DELETE FROM revision_certs"));
2541}
2542
2543void
2544database::delete_existing_manifests()
2545{
2546 imp->execute(query("DELETE FROM manifests"));
2547 imp->execute(query("DELETE FROM manifest_deltas"));
2548}
2549
2550void
2551database::delete_existing_rosters()
2552{
2553 imp->execute(query("DELETE FROM rosters"));
2554 imp->execute(query("DELETE FROM roster_deltas"));
2555 imp->execute(query("DELETE FROM next_roster_node_number"));
2556}
2557
2558void
2559database::delete_existing_heights()
2560{
2561 imp->execute(query("DELETE FROM heights"));
2562}
2563
2564/// Deletes one revision from the local database.
2565/// @see kill_rev_locally
2566void
2567database::delete_existing_rev_and_certs(revision_id const & rid)
2568{
2569 transaction_guard guard (*this);
2570
2571 // Check that the revision exists and doesn't have any children.
2572 I(revision_exists(rid));
2573 set<revision_id> children;
2574 get_revision_children(rid, children);
2575 I(children.empty());
2576
2577
2578 L(FL("Killing revision %s locally") % rid);
2579
2580 // Kill the certs, ancestry, and revision.
2581 imp->execute(query("DELETE from revision_certs WHERE id = ?")
2582 % blob(rid.inner()()));
2583 imp->cert_stamper.note_change();
2584
2585 imp->execute(query("DELETE from revision_ancestry WHERE child = ?")
2586 % blob(rid.inner()()));
2587
2588 imp->execute(query("DELETE from heights WHERE revision = ?")
2589 % blob(rid.inner()()));
2590
2591 imp->execute(query("DELETE from revisions WHERE id = ?")
2592 % blob(rid.inner()()));
2593
2594 guard.commit();
2595}
2596
2597/// Deletes all certs referring to a particular branch.
2598void
2599database::delete_branch_named(cert_value const & branch)
2600{
2601 L(FL("Deleting all references to branch %s") % branch);
2602 imp->execute(query("DELETE FROM revision_certs WHERE name='branch' AND value =?")
2603 % blob(branch()));
2604 imp->cert_stamper.note_change();
2605 imp->execute(query("DELETE FROM branch_epochs WHERE branch=?")
2606 % blob(branch()));
2607}
2608
2609/// Deletes all certs referring to a particular tag.
2610void
2611database::delete_tag_named(cert_value const & tag)
2612{
2613 L(FL("Deleting all references to tag %s") % tag);
2614 imp->execute(query("DELETE FROM revision_certs WHERE name='tag' AND value =?")
2615 % blob(tag()));
2616 imp->cert_stamper.note_change();
2617}
2618
2619// crypto key management
2620
2621void
2622database::get_key_ids(vector<rsa_keypair_id> & pubkeys)
2623{
2624 pubkeys.clear();
2625 results res;
2626
2627 imp->fetch(res, one_col, any_rows, query("SELECT id FROM public_keys"));
2628
2629 for (size_t i = 0; i < res.size(); ++i)
2630 pubkeys.push_back(rsa_keypair_id(res[i][0]));
2631}
2632
2633void
2634database::get_key_ids(globish const & pattern,
2635 vector<rsa_keypair_id> & pubkeys)
2636{
2637 pubkeys.clear();
2638 results res;
2639
2640 imp->fetch(res, one_col, any_rows, query("SELECT id FROM public_keys"));
2641
2642 for (size_t i = 0; i < res.size(); ++i)
2643 if (pattern.matches(res[i][0]))
2644 pubkeys.push_back(rsa_keypair_id(res[i][0]));
2645}
2646
2647void
2648database_impl::get_keys(string const & table, vector<rsa_keypair_id> & keys)
2649{
2650 keys.clear();
2651 results res;
2652 fetch(res, one_col, any_rows, query("SELECT id FROM " + table));
2653 for (size_t i = 0; i < res.size(); ++i)
2654 keys.push_back(rsa_keypair_id(res[i][0]));
2655}
2656
2657void
2658database::get_public_keys(vector<rsa_keypair_id> & keys)
2659{
2660 imp->get_keys("public_keys", keys);
2661}
2662
2663bool
2664database::public_key_exists(id const & hash)
2665{
2666 results res;
2667 imp->fetch(res, one_col, any_rows,
2668 query("SELECT id FROM public_keys WHERE hash = ?")
2669 % blob(hash()));
2670 I((res.size() == 1) || (res.empty()));
2671 if (res.size() == 1)
2672 return true;
2673 return false;
2674}
2675
2676bool
2677database::public_key_exists(rsa_keypair_id const & id)
2678{
2679 results res;
2680 imp->fetch(res, one_col, any_rows,
2681 query("SELECT id FROM public_keys WHERE id = ?")
2682 % text(id()));
2683 I((res.size() == 1) || (res.empty()));
2684 if (res.size() == 1)
2685 return true;
2686 return false;
2687}
2688
2689void
2690database::get_pubkey(id const & hash,
2691 rsa_keypair_id & id,
2692 rsa_pub_key & pub)
2693{
2694 results res;
2695 imp->fetch(res, 2, one_row,
2696 query("SELECT id, keydata FROM public_keys WHERE hash = ?")
2697 % blob(hash()));
2698 id = rsa_keypair_id(res[0][0]);
2699 pub = rsa_pub_key(res[0][1]);
2700}
2701
2702void
2703database::get_key(rsa_keypair_id const & pub_id,
2704 rsa_pub_key & pub)
2705{
2706 results res;
2707 imp->fetch(res, one_col, one_row,
2708 query("SELECT keydata FROM public_keys WHERE id = ?")
2709 % text(pub_id()));
2710 pub = rsa_pub_key(res[0][0]);
2711}
2712
2713bool
2714database::put_key(rsa_keypair_id const & pub_id,
2715 rsa_pub_key const & pub)
2716{
2717 if (public_key_exists(pub_id))
2718 {
2719 rsa_pub_key tmp;
2720 get_key(pub_id, tmp);
2721 if (!keys_match(pub_id, tmp, pub_id, pub))
2722 W(F("key '%s' is not equal to key '%s' in database") % pub_id % pub_id);
2723 L(FL("skipping existing public key %s") % pub_id);
2724 return false;
2725 }
2726
2727 L(FL("putting public key %s") % pub_id);
2728
2729 id thash;
2730 key_hash_code(pub_id, pub, thash);
2731 I(!public_key_exists(thash));
2732
2733 imp->execute(query("INSERT INTO public_keys VALUES(?, ?, ?)")
2734 % blob(thash())
2735 % text(pub_id())
2736 % blob(pub()));
2737
2738 return true;
2739}
2740
2741void
2742database::delete_public_key(rsa_keypair_id const & pub_id)
2743{
2744 imp->execute(query("DELETE FROM public_keys WHERE id = ?")
2745 % text(pub_id()));
2746}
2747
2748void
2749database::encrypt_rsa(rsa_keypair_id const & pub_id,
2750 string const & plaintext,
2751 rsa_oaep_sha_data & ciphertext)
2752{
2753 rsa_pub_key pub;
2754 get_key(pub_id, pub);
2755
2756 SecureVector<Botan::byte> pub_block;
2757 pub_block.set(reinterpret_cast<Botan::byte const *>(pub().data()),
2758 pub().size());
2759
2760 shared_ptr<X509_PublicKey> x509_key(Botan::X509::load_key(pub_block));
2761 shared_ptr<RSA_PublicKey> pub_key
2762 = shared_dynamic_cast<RSA_PublicKey>(x509_key);
2763 if (!pub_key)
2764 throw informative_failure("Failed to get RSA encrypting key");
2765
2766 shared_ptr<PK_Encryptor>
2767 encryptor(get_pk_encryptor(*pub_key, "EME1(SHA-1)"));
2768
2769 SecureVector<Botan::byte> ct;
2770 ct = encryptor->encrypt(
2771 reinterpret_cast<Botan::byte const *>(plaintext.data()),
2772 plaintext.size(), *rng);
2773 ciphertext = rsa_oaep_sha_data(string(reinterpret_cast<char const *>(ct.begin()),
2774 ct.size()));
2775}
2776
2777cert_status
2778database::check_signature(rsa_keypair_id const & id,
2779 string const & alleged_text,
2780 rsa_sha1_signature const & signature)
2781{
2782 shared_ptr<PK_Verifier> verifier;
2783
2784 verifier_cache::const_iterator i = imp->verifiers.find(id);
2785 if (i != imp->verifiers.end())
2786 verifier = i->second.first;
2787
2788 else
2789 {
2790 rsa_pub_key pub;
2791 SecureVector<Botan::byte> pub_block;
2792
2793 if (!public_key_exists(id))
2794 return cert_unknown;
2795
2796 get_key(id, pub);
2797 pub_block.set(reinterpret_cast<Botan::byte const *>(pub().data()),
2798 pub().size());
2799
2800 L(FL("building verifier for %d-byte pub key") % pub_block.size());
2801 shared_ptr<X509_PublicKey> x509_key(Botan::X509::load_key(pub_block));
2802 shared_ptr<RSA_PublicKey> pub_key
2803 = boost::shared_dynamic_cast<RSA_PublicKey>(x509_key);
2804
2805 E(pub_key,
2806 F("Failed to get RSA verifying key for %s") % id);
2807
2808 verifier.reset(get_pk_verifier(*pub_key, "EMSA3(SHA-1)"));
2809
2810 /* XXX This is ugly. We need to keep the key around
2811 * as long as the verifier is around, but the shared_ptr will go
2812 * away after we leave this scope. Hence we store a pair of
2813 * <verifier,key> so they both exist. */
2814 imp->verifiers.insert(make_pair(id, make_pair(verifier, pub_key)));
2815 }
2816
2817 // check the text+sig against the key
2818 L(FL("checking %d-byte signature") % signature().size());
2819
2820 if (verifier->verify_message(
2821 reinterpret_cast<Botan::byte const*>(alleged_text.data()),
2822 alleged_text.size(),
2823 reinterpret_cast<Botan::byte const*>(signature().data()),
2824 signature().size()))
2825 return cert_ok;
2826 else
2827 return cert_bad;
2828}
2829
2830// cert management
2831
2832bool
2833database_impl::cert_exists(cert const & t,
2834 string const & table)
2835{
2836 results res;
2837 query q = query("SELECT id FROM " + table + " WHERE id = ? "
2838 "AND name = ? "
2839 "AND value = ? "
2840 "AND keypair = ? "
2841 "AND signature = ?")
2842 % blob(t.ident.inner()())
2843 % text(t.name())
2844 % blob(t.value())
2845 % text(t.key())
2846 % blob(t.sig());
2847
2848 fetch(res, 1, any_rows, q);
2849
2850 I(res.empty() || res.size() == 1);
2851 return res.size() == 1;
2852}
2853
2854void
2855database_impl::put_cert(cert const & t,
2856 string const & table)
2857{
2858 id thash;
2859 cert_hash_code(t, thash);
2860 rsa_sha1_signature sig;
2861
2862 string insert = "INSERT INTO " + table + " VALUES(?, ?, ?, ?, ?, ?)";
2863
2864 execute(query(insert)
2865 % blob(thash())
2866 % blob(t.ident.inner()())
2867 % text(t.name())
2868 % blob(t.value())
2869 % text(t.key())
2870 % blob(t.sig()));
2871}
2872
2873void
2874database_impl::results_to_certs(results const & res,
2875 vector<cert> & certs)
2876{
2877 certs.clear();
2878 for (size_t i = 0; i < res.size(); ++i)
2879 {
2880 cert t;
2881 t = cert(revision_id(res[i][0]),
2882 cert_name(res[i][1]),
2883 cert_value(res[i][2]),
2884 rsa_keypair_id(res[i][3]),
2885 rsa_sha1_signature(res[i][4]));
2886 certs.push_back(t);
2887 }
2888}
2889
2890void
2891database_impl::install_functions()
2892{
2893 // register any functions we're going to use
2894 I(sqlite3_create_function(sql(), "gunzip", -1,
2895 SQLITE_UTF8, NULL,
2896 &sqlite3_gunzip_fn,
2897 NULL, NULL) == 0);
2898}
2899
2900void
2901database_impl::get_certs(vector<cert> & certs,
2902 string const & table)
2903{
2904 results res;
2905 query q("SELECT id, name, value, keypair, signature FROM " + table);
2906 fetch(res, 5, any_rows, q);
2907 results_to_certs(res, certs);
2908}
2909
2910
2911void
2912database_impl::get_certs(id const & ident,
2913 vector<cert> & certs,
2914 string const & table)
2915{
2916 results res;
2917 query q("SELECT id, name, value, keypair, signature FROM " + table +
2918 " WHERE id = ?");
2919
2920 fetch(res, 5, any_rows, q % blob(ident()));
2921 results_to_certs(res, certs);
2922}
2923
2924
2925void
2926database_impl::get_certs(cert_name const & name,
2927 vector<cert> & certs,
2928 string const & table)
2929{
2930 results res;
2931 query q("SELECT id, name, value, keypair, signature FROM " + table +
2932 " WHERE name = ?");
2933 fetch(res, 5, any_rows, q % text(name()));
2934 results_to_certs(res, certs);
2935}
2936
2937
2938void
2939database_impl::get_certs(id const & ident,
2940 cert_name const & name,
2941 vector<cert> & certs,
2942 string const & table)
2943{
2944 results res;
2945 query q("SELECT id, name, value, keypair, signature FROM " + table +
2946 " WHERE id = ? AND name = ?");
2947
2948 fetch(res, 5, any_rows,
2949 q % blob(ident())
2950 % text(name()));
2951 results_to_certs(res, certs);
2952}
2953
2954void
2955database_impl::get_certs(cert_name const & name,
2956 cert_value const & val,
2957 vector<cert> & certs,
2958 string const & table)
2959{
2960 results res;
2961 query q("SELECT id, name, value, keypair, signature FROM " + table +
2962 " WHERE name = ? AND value = ?");
2963
2964 fetch(res, 5, any_rows,
2965 q % text(name())
2966 % blob(val()));
2967 results_to_certs(res, certs);
2968}
2969
2970
2971void
2972database_impl::get_certs(id const & ident,
2973 cert_name const & name,
2974 cert_value const & value,
2975 vector<cert> & certs,
2976 string const & table)
2977{
2978 results res;
2979 query q("SELECT id, name, value, keypair, signature FROM " + table +
2980 " WHERE id = ? AND name = ? AND value = ?");
2981
2982 fetch(res, 5, any_rows,
2983 q % blob(ident())
2984 % text(name())
2985 % blob(value()));
2986 results_to_certs(res, certs);
2987}
2988
2989
2990
2991bool
2992database::revision_cert_exists(revision<cert> const & cert)
2993{
2994 return imp->cert_exists(cert.inner(), "revision_certs");
2995}
2996
2997bool
2998database::put_revision_cert(revision<cert> const & cert)
2999{
3000 if (revision_cert_exists(cert))
3001 {
3002 L(FL("revision cert on '%s' already exists in db")
3003 % cert.inner().ident);
3004 return false;
3005 }
3006
3007 if (!revision_exists(revision_id(cert.inner().ident)))
3008 {
3009 W(F("cert revision '%s' does not exist in db")
3010 % cert.inner().ident);
3011 W(F("dropping cert"));
3012 return false;
3013 }
3014
3015 imp->put_cert(cert.inner(), "revision_certs");
3016 imp->cert_stamper.note_change();
3017 return true;
3018}
3019
3020outdated_indicator
3021database::get_revision_cert_nobranch_index(vector< pair<revision_id,
3022 pair<revision_id, rsa_keypair_id> > > & idx)
3023{
3024 // share some storage
3025 id::symtab id_syms;
3026
3027 results res;
3028 imp->fetch(res, 3, any_rows,
3029 query("SELECT hash, id, keypair "
3030 "FROM 'revision_certs' WHERE name != 'branch'"));
3031
3032 idx.clear();
3033 idx.reserve(res.size());
3034 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3035 {
3036 idx.push_back(make_pair(revision_id((*i)[0]),
3037 make_pair(revision_id((*i)[1]),
3038 rsa_keypair_id((*i)[2]))));
3039 }
3040 return imp->cert_stamper.get_indicator();
3041}
3042
3043outdated_indicator
3044database::get_revision_certs(vector< revision<cert> > & ts)
3045{
3046 vector<cert> certs;
3047 imp->get_certs(certs, "revision_certs");
3048 ts.clear();
3049 add_decoration_to_container(certs, ts);
3050 return imp->cert_stamper.get_indicator();
3051}
3052
3053outdated_indicator
3054database::get_revision_certs(cert_name const & name,
3055 vector< revision<cert> > & ts)
3056{
3057 vector<cert> certs;
3058 imp->get_certs(name, certs, "revision_certs");
3059 ts.clear();
3060 add_decoration_to_container(certs, ts);
3061 return imp->cert_stamper.get_indicator();
3062}
3063
3064outdated_indicator
3065database::get_revision_certs(revision_id const & id,
3066 cert_name const & name,
3067 vector< revision<cert> > & ts)
3068{
3069 vector<cert> certs;
3070 imp->get_certs(id.inner(), name, certs, "revision_certs");
3071 ts.clear();
3072 add_decoration_to_container(certs, ts);
3073 return imp->cert_stamper.get_indicator();
3074}
3075
3076outdated_indicator
3077database::get_revision_certs(revision_id const & id,
3078 cert_name const & name,
3079 cert_value const & val,
3080 vector< revision<cert> > & ts)
3081{
3082 vector<cert> certs;
3083 imp->get_certs(id.inner(), name, val, certs, "revision_certs");
3084 ts.clear();
3085 add_decoration_to_container(certs, ts);
3086 return imp->cert_stamper.get_indicator();
3087}
3088
3089outdated_indicator
3090database::get_revisions_with_cert(cert_name const & name,
3091 cert_value const & val,
3092 set<revision_id> & revisions)
3093{
3094 revisions.clear();
3095 results res;
3096 query q("SELECT id FROM revision_certs WHERE name = ? AND value = ?");
3097 imp->fetch(res, one_col, any_rows, q % text(name()) % blob(val()));
3098 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3099 revisions.insert(revision_id((*i)[0]));
3100 return imp->cert_stamper.get_indicator();
3101}
3102
3103outdated_indicator
3104database::get_revision_certs(cert_name const & name,
3105 cert_value const & val,
3106 vector< revision<cert> > & ts)
3107{
3108 vector<cert> certs;
3109 imp->get_certs(name, val, certs, "revision_certs");
3110 ts.clear();
3111 add_decoration_to_container(certs, ts);
3112 return imp->cert_stamper.get_indicator();
3113}
3114
3115outdated_indicator
3116database::get_revision_certs(revision_id const & id,
3117 vector< revision<cert> > & ts)
3118{
3119 vector<cert> certs;
3120 imp->get_certs(id.inner(), certs, "revision_certs");
3121 ts.clear();
3122 add_decoration_to_container(certs, ts);
3123 return imp->cert_stamper.get_indicator();
3124}
3125
3126outdated_indicator
3127database::get_revision_certs(revision_id const & ident,
3128 vector<id> & ts)
3129{
3130 results res;
3131 vector<cert> certs;
3132 imp->fetch(res, one_col, any_rows,
3133 query("SELECT hash "
3134 "FROM revision_certs "
3135 "WHERE id = ?")
3136 % blob(ident.inner()()));
3137 ts.clear();
3138 for (size_t i = 0; i < res.size(); ++i)
3139 ts.push_back(id(res[i][0]));
3140 return imp->cert_stamper.get_indicator();
3141}
3142
3143void
3144database::get_revision_cert(id const & hash,
3145 revision<cert> & c)
3146{
3147 results res;
3148 vector<cert> certs;
3149 imp->fetch(res, 5, one_row,
3150 query("SELECT id, name, value, keypair, signature "
3151 "FROM revision_certs "
3152 "WHERE hash = ?")
3153 % blob(hash()));
3154 imp->results_to_certs(res, certs);
3155 I(certs.size() == 1);
3156 c = revision<cert>(certs[0]);
3157}
3158
3159bool
3160database::revision_cert_exists(revision_id const & hash)
3161{
3162 results res;
3163 vector<cert> certs;
3164 imp->fetch(res, one_col, any_rows,
3165 query("SELECT id "
3166 "FROM revision_certs "
3167 "WHERE hash = ?")
3168 % blob(hash.inner()()));
3169 I(res.empty() || res.size() == 1);
3170 return (res.size() == 1);
3171}
3172
3173void
3174database::get_manifest_certs(manifest_id const & id,
3175 vector< manifest<cert> > & ts)
3176{
3177 vector<cert> certs;
3178 imp->get_certs(id.inner(), certs, "manifest_certs");
3179 ts.clear();
3180 add_decoration_to_container(certs, ts);
3181}
3182
3183
3184void
3185database::get_manifest_certs(cert_name const & name,
3186 vector< manifest<cert> > & ts)
3187{
3188 vector<cert> certs;
3189 imp->get_certs(name, certs, "manifest_certs");
3190 ts.clear();
3191 add_decoration_to_container(certs, ts);
3192}
3193
3194
3195// completions
3196void
3197database_impl::add_prefix_matching_constraint(string const & colname,
3198 string const & prefix,
3199 query & q)
3200{
3201 L(FL("add_prefix_matching_constraint for '%s'") % prefix);
3202
3203 if (prefix.empty())
3204 q.sql_cmd += "1"; // always true
3205 else
3206 {
3207 string binary_prefix = decode_hexenc(prefix);
3208 string lower_bound(binary_prefix);
3209 string upper_bound(binary_prefix);
3210
3211 string::reverse_iterator ity(upper_bound.rbegin());
3212 if (ity != upper_bound.rend())
3213 ++(*ity);
3214 while ((*ity == 0) && ity != upper_bound.rend())
3215 {
3216 ++ity;
3217 ++(*ity);
3218 }
3219
3220 if (ity == upper_bound.rend())
3221 {
3222 // no upper bound needed, as the lower bound is
3223 // 0xffffff...
3224 if (global_sanity.debug_p())
3225 L(FL("prefix_matcher: only lower bound ('%s')")
3226 % encode_hexenc(lower_bound));
3227
3228 q.sql_cmd += colname + " > ?";
3229 q.args.push_back(blob(lower_bound));
3230 }
3231 else
3232 {
3233 if (global_sanity.debug_p())
3234 L(FL("prefix_matcher: lower bound ('%s') and upper bound ('%s')")
3235 % encode_hexenc(lower_bound)
3236 % encode_hexenc(upper_bound));
3237
3238 q.sql_cmd += colname + " BETWEEN ? AND ?";
3239 q.args.push_back(blob(lower_bound));
3240 q.args.push_back(blob(upper_bound));
3241 }
3242
3243 // encode_hexenc might have lost a nibble a the end, thus we possibly
3244 // need to add a second check, with a LIKE operator on the hex
3245 // encoded string.
3246
3247 if (prefix.size() % 2 == 1)
3248 {
3249 string pattern = prefix + '%';
3250 q.sql_cmd += " AND (hex(" + colname + ") LIKE ?)";
3251 q.args.push_back(text(pattern));
3252 }
3253 }
3254}
3255
3256void
3257database::complete(string const & partial,
3258 set<revision_id> & completions)
3259{
3260 results res;
3261 completions.clear();
3262 query q("SELECT id FROM revisions WHERE ");
3263
3264 imp->add_prefix_matching_constraint("id", partial, q);
3265 imp->fetch(res, 1, any_rows, q);
3266
3267 for (size_t i = 0; i < res.size(); ++i)
3268 completions.insert(revision_id(res[i][0]));
3269}
3270
3271
3272void
3273database::complete(string const & partial,
3274 set<file_id> & completions)
3275{
3276 results res;
3277 completions.clear();
3278
3279 query q("SELECT id FROM files WHERE ");
3280 imp->add_prefix_matching_constraint("id", partial, q);
3281 imp->fetch(res, 1, any_rows, q);
3282
3283 for (size_t i = 0; i < res.size(); ++i)
3284 completions.insert(file_id(res[i][0]));
3285
3286 res.clear();
3287
3288 q = query("SELECT id FROM file_deltas WHERE ");
3289 imp->add_prefix_matching_constraint("id", partial, q);
3290 imp->fetch(res, 1, any_rows, q);
3291
3292 for (size_t i = 0; i < res.size(); ++i)
3293 completions.insert(file_id(res[i][0]));
3294}
3295
3296void
3297database::complete(string const & partial,
3298 set< pair<key_id, utf8 > > & completions)
3299{
3300 results res;
3301 completions.clear();
3302 query q("SELECT hash, id FROM public_keys WHERE ");
3303
3304 imp->add_prefix_matching_constraint("hash", partial, q);
3305 imp->fetch(res, 2, any_rows, q);
3306
3307 for (size_t i = 0; i < res.size(); ++i)
3308 completions.insert(make_pair(key_id(res[i][0]),
3309 utf8(res[i][1])));
3310}
3311
3312// revision selectors
3313
3314void
3315database::select_parent(string const & partial,
3316 set<revision_id> & completions)
3317{
3318 results res;
3319 completions.clear();
3320
3321 query q("SELECT DISTINCT parent FROM revision_ancestry WHERE ");
3322 imp->add_prefix_matching_constraint("child", encode_hexenc(partial), q);
3323 imp->fetch(res, 1, any_rows, q);
3324
3325 for (size_t i = 0; i < res.size(); ++i)
3326 completions.insert(revision_id(res[i][0]));
3327}
3328
3329void
3330database::select_cert(string const & certname,
3331 set<revision_id> & completions)
3332{
3333 results res;
3334 completions.clear();
3335
3336 imp->fetch(res, 1, any_rows,
3337 query("SELECT DISTINCT id FROM revision_certs WHERE name = ?")
3338 % text(certname));
3339
3340 for (size_t i = 0; i < res.size(); ++i)
3341 completions.insert(revision_id(res[i][0]));
3342}
3343
3344void
3345database::select_cert(string const & certname, string const & certvalue,
3346 set<revision_id> & completions)
3347{
3348 results res;
3349 completions.clear();
3350
3351 imp->fetch(res, 1, any_rows,
3352 query("SELECT DISTINCT id FROM revision_certs"
3353 " WHERE name = ? AND CAST(value AS TEXT) GLOB ?")
3354 % text(certname) % text(certvalue));
3355
3356 for (size_t i = 0; i < res.size(); ++i)
3357 completions.insert(revision_id(res[i][0]));
3358}
3359
3360void
3361database::select_author_tag_or_branch(string const & partial,
3362 set<revision_id> & completions)
3363{
3364 results res;
3365 completions.clear();
3366
3367 string pattern = partial + "*";
3368
3369 imp->fetch(res, 1, any_rows,
3370 query("SELECT DISTINCT id FROM revision_certs"
3371 " WHERE (name=? OR name=? OR name=?)"
3372 " AND CAST(value AS TEXT) GLOB ?")
3373 % text(author_cert_name()) % text(tag_cert_name())
3374 % text(branch_cert_name()) % text(pattern));
3375
3376 for (size_t i = 0; i < res.size(); ++i)
3377 completions.insert(revision_id(res[i][0]));
3378}
3379
3380void
3381database::select_date(string const & date, string const & comparison,
3382 set<revision_id> & completions)
3383{
3384 results res;
3385 completions.clear();
3386
3387 query q;
3388 q.sql_cmd = ("SELECT DISTINCT id FROM revision_certs "
3389 "WHERE name = ? AND CAST(value AS TEXT) ");
3390 q.sql_cmd += comparison;
3391 q.sql_cmd += " ?";
3392
3393 imp->fetch(res, 1, any_rows,
3394 q % text(date_cert_name()) % text(date));
3395 for (size_t i = 0; i < res.size(); ++i)
3396 completions.insert(revision_id(res[i][0]));
3397}
3398
3399// epochs
3400
3401void
3402database::get_epochs(map<branch_name, epoch_data> & epochs)
3403{
3404 epochs.clear();
3405 results res;
3406 imp->fetch(res, 2, any_rows, query("SELECT branch, epoch FROM branch_epochs"));
3407 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3408 {
3409 branch_name decoded(idx(*i, 0));
3410 I(epochs.find(decoded) == epochs.end());
3411 epochs.insert(make_pair(decoded,
3412 epoch_data(idx(*i, 1))));
3413 }
3414}
3415
3416void
3417database::get_epoch(epoch_id const & eid,
3418 branch_name & branch, epoch_data & epo)
3419{
3420 I(epoch_exists(eid));
3421 results res;
3422 imp->fetch(res, 2, any_rows,
3423 query("SELECT branch, epoch FROM branch_epochs"
3424 " WHERE hash = ?")
3425 % blob(eid.inner()()));
3426 I(res.size() == 1);
3427 branch = branch_name(idx(idx(res, 0), 0));
3428 epo = epoch_data(idx(idx(res, 0), 1));
3429}
3430
3431bool
3432database::epoch_exists(epoch_id const & eid)
3433{
3434 results res;
3435 imp->fetch(res, one_col, any_rows,
3436 query("SELECT hash FROM branch_epochs WHERE hash = ?")
3437 % blob(eid.inner()()));
3438 I(res.size() == 1 || res.empty());
3439 return res.size() == 1;
3440}
3441
3442void
3443database::set_epoch(branch_name const & branch, epoch_data const & epo)
3444{
3445 epoch_id eid;
3446 epoch_hash_code(branch, epo, eid);
3447 I(epo.inner()().size() == constants::epochlen_bytes);
3448 imp->execute(query("INSERT OR REPLACE INTO branch_epochs VALUES(?, ?, ?)")
3449 % blob(eid.inner()())
3450 % blob(branch())
3451 % blob(epo.inner()()));
3452}
3453
3454void
3455database::clear_epoch(branch_name const & branch)
3456{
3457 imp->execute(query("DELETE FROM branch_epochs WHERE branch = ?")
3458 % blob(branch()));
3459}
3460
3461bool
3462database::check_integrity()
3463{
3464 results res;
3465 imp->fetch(res, one_col, any_rows, query("PRAGMA integrity_check"));
3466 I(res.size() == 1);
3467 I(res[0].size() == 1);
3468
3469 return res[0][0] == "ok";
3470}
3471
3472// vars
3473
3474void
3475database::get_vars(map<var_key, var_value> & vars)
3476{
3477 vars.clear();
3478 results res;
3479 imp->fetch(res, 3, any_rows, query("SELECT domain, name, value FROM db_vars"));
3480 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3481 {
3482 var_domain domain(idx(*i, 0));
3483 var_name name(idx(*i, 1));
3484 var_value value(idx(*i, 2));
3485 I(vars.find(make_pair(domain, name)) == vars.end());
3486 vars.insert(make_pair(make_pair(domain, name), value));
3487 }
3488}
3489
3490void
3491database::get_var(var_key const & key, var_value & value)
3492{
3493 // FIXME: sillyly inefficient. Doesn't really matter, though.
3494 map<var_key, var_value> vars;
3495 get_vars(vars);
3496 map<var_key, var_value>::const_iterator i = vars.find(key);
3497 I(i != vars.end());
3498 value = i->second;
3499}
3500
3501bool
3502database::var_exists(var_key const & key)
3503{
3504 // FIXME: sillyly inefficient. Doesn't really matter, though.
3505 map<var_key, var_value> vars;
3506 get_vars(vars);
3507 map<var_key, var_value>::const_iterator i = vars.find(key);
3508 return i != vars.end();
3509}
3510
3511void
3512database::set_var(var_key const & key, var_value const & value)
3513{
3514 imp->execute(query("INSERT OR REPLACE INTO db_vars VALUES(?, ?, ?)")
3515 % text(key.first())
3516 % blob(key.second())
3517 % blob(value()));
3518}
3519
3520void
3521database::clear_var(var_key const & key)
3522{
3523 imp->execute(query("DELETE FROM db_vars WHERE domain = ? AND name = ?")
3524 % text(key.first())
3525 % blob(key.second()));
3526}
3527
3528// branches
3529
3530outdated_indicator
3531database::get_branches(vector<string> & names)
3532{
3533 results res;
3534 query q("SELECT DISTINCT value FROM revision_certs WHERE name = ?");
3535 string cert_name = "branch";
3536 imp->fetch(res, one_col, any_rows, q % text(cert_name));
3537 for (size_t i = 0; i < res.size(); ++i)
3538 {
3539 names.push_back(res[i][0]);
3540 }
3541 return imp->cert_stamper.get_indicator();
3542}
3543
3544outdated_indicator
3545database::get_branches(globish const & glob,
3546 vector<string> & names)
3547{
3548 results res;
3549 query q("SELECT DISTINCT value FROM revision_certs WHERE name = ?");
3550 string cert_name = "branch";
3551 imp->fetch(res, one_col, any_rows, q % text(cert_name));
3552 for (size_t i = 0; i < res.size(); ++i)
3553 {
3554 if (glob.matches(res[i][0]))
3555 names.push_back(res[i][0]);
3556 }
3557 return imp->cert_stamper.get_indicator();
3558}
3559
3560void
3561database::get_roster(revision_id const & rev_id,
3562 roster_t & roster)
3563{
3564 marking_map mm;
3565 get_roster(rev_id, roster, mm);
3566}
3567
3568void
3569database::get_roster(revision_id const & rev_id,
3570 roster_t & roster,
3571 marking_map & marking)
3572{
3573 if (rev_id.inner()().empty())
3574 {
3575 roster = roster_t();
3576 marking = marking_map();
3577 return;
3578 }
3579
3580 cached_roster cr;
3581 get_roster(rev_id, cr);
3582 roster = *cr.first;
3583 marking = *cr.second;
3584}
3585
3586void
3587database::get_roster(revision_id const & rev_id, cached_roster & cr)
3588{
3589 get_roster_version(rev_id, cr);
3590 I(cr.first);
3591 I(cr.second);
3592}
3593
3594void
3595database::put_roster(revision_id const & rev_id,
3596 roster_t_cp const & roster,
3597 marking_map_cp const & marking)
3598{
3599 I(roster);
3600 I(marking);
3601 MM(rev_id);
3602
3603 transaction_guard guard(*this);
3604
3605 // Our task is to add this roster, and deltify all the incoming edges (if
3606 // they aren't already).
3607
3608 imp->roster_cache.insert_dirty(rev_id, make_pair(roster, marking));
3609
3610 set<revision_id> parents;
3611 get_revision_parents(rev_id, parents);
3612
3613 // Now do what deltify would do if we bothered
3614 for (set<revision_id>::const_iterator i = parents.begin();
3615 i != parents.end(); ++i)
3616 {
3617 if (null_id(*i))
3618 continue;
3619 revision_id old_rev = *i;
3620 if (imp->roster_base_stored(old_rev))
3621 {
3622 cached_roster cr;
3623 get_roster_version(old_rev, cr);
3624 roster_delta reverse_delta;
3625 delta_rosters(*roster, *marking, *(cr.first), *(cr.second), reverse_delta);
3626 if (imp->roster_cache.exists(old_rev))
3627 imp->roster_cache.mark_clean(old_rev);
3628 imp->drop(old_rev.inner(), "rosters");
3629 imp->put_roster_delta(old_rev, rev_id, reverse_delta);
3630 }
3631 }
3632 guard.commit();
3633}
3634
3635// for get_uncommon_ancestors
3636struct rev_height_graph : rev_graph
3637{
3638 rev_height_graph(database & db) : db(db) {}
3639 virtual void get_parents(revision_id const & rev, set<revision_id> & parents) const
3640 {
3641 db.get_revision_parents(rev, parents);
3642 }
3643 virtual void get_children(revision_id const & rev, set<revision_id> & parents) const
3644 {
3645 // not required
3646 I(false);
3647 }
3648 virtual void get_height(revision_id const & rev, rev_height & h) const
3649 {
3650 db.get_rev_height(rev, h);
3651 }
3652
3653 database & db;
3654};
3655
3656void
3657database::get_uncommon_ancestors(revision_id const & a,
3658 revision_id const & b,
3659 set<revision_id> & a_uncommon_ancs,
3660 set<revision_id> & b_uncommon_ancs)
3661{
3662
3663 rev_height_graph graph(*this);
3664 ::get_uncommon_ancestors(a, b, graph, a_uncommon_ancs, b_uncommon_ancs);
3665}
3666
3667node_id
3668database::next_node_id()
3669{
3670 transaction_guard guard(*this);
3671 results res;
3672
3673 // We implement this as a fixed db var.
3674 imp->fetch(res, one_col, any_rows,
3675 query("SELECT node FROM next_roster_node_number"));
3676
3677 u64 n = 1;
3678 if (res.empty())
3679 {
3680 imp->execute(query("INSERT INTO next_roster_node_number VALUES(1)"));
3681 }
3682 else
3683 {
3684 I(res.size() == 1);
3685 n = lexical_cast<u64>(res[0][0]);
3686 ++n;
3687 imp->execute(query("UPDATE next_roster_node_number SET node = ?")
3688 % text(lexical_cast<string>(n)));
3689 }
3690 guard.commit();
3691 return static_cast<node_id>(n);
3692}
3693
3694void
3695database_impl::check_filename()
3696{
3697 N(!filename.empty(), F("no database specified"));
3698}
3699
3700
3701void
3702database_impl::check_db_exists()
3703{
3704 switch (get_path_status(filename))
3705 {
3706 case path::file:
3707 return;
3708
3709 case path::nonexistent:
3710 N(false, F("database %s does not exist") % filename);
3711
3712 case path::directory:
3713 if (directory_is_workspace(filename))
3714 {
3715 system_path workspace_database;
3716 workspace::get_database_option(filename, workspace_database);
3717 N(workspace_database.empty(),
3718 F("%s is a workspace, not a database\n"
3719 "(did you mean %s?)") % filename % workspace_database);
3720 }
3721 N(false, F("%s is a directory, not a database") % filename);
3722 }
3723}
3724
3725void
3726database_impl::check_db_nonexistent()
3727{
3728 require_path_is_nonexistent(filename,
3729 F("database %s already exists")
3730 % filename);
3731
3732 system_path journal(filename.as_internal() + "-journal");
3733 require_path_is_nonexistent(journal,
3734 F("existing (possibly stale) journal file '%s' "
3735 "has same stem as new database '%s'\n"
3736 "cancelling database creation")
3737 % journal % filename);
3738
3739}
3740
3741void
3742database_impl::open()
3743{
3744 I(!__sql);
3745
3746 if (sqlite3_open(filename.as_external().c_str(), &__sql) == SQLITE_NOMEM)
3747 throw std::bad_alloc();
3748
3749 I(__sql);
3750 assert_sqlite3_ok(__sql);
3751}
3752
3753void
3754database_impl::close()
3755{
3756 I(__sql);
3757
3758 sqlite3_close(__sql);
3759 __sql = 0;
3760
3761 I(!__sql);
3762}
3763
3764// the database holds onto the lua_hooks object and uses it to re-expose
3765// these two hooks. it is impractical to pass the lua_hooks object down to
3766// all the places where this is used, and they're all going to get
3767// reexamined when we do policy branches anyway. also, arguably this is
3768// cleaner, because those places don't have access to *all* the lua hooks,
3769// just these two. (manifest cert trust is only relevant to pre-roster
3770// migration, but revision cert trust comes up everywhere erase_bogus_certs
3771// is called.) (A near-term refactor that might make sense: make
3772// erase_bogus_certs a project_t member and have the project_t hold the
3773// lua_hooks reference.)
3774
3775bool
3776database::hook_get_manifest_cert_trust(set<rsa_keypair_id> const & signers,
3777 manifest_id const & id, cert_name const & name, cert_value const & val)
3778{
3779 return lua.hook_get_manifest_cert_trust(signers, id, name, val);
3780};
3781
3782bool
3783database::hook_get_revision_cert_trust(set<rsa_keypair_id> const & signers,
3784 revision_id const & id, cert_name const & name, cert_value const & val)
3785{
3786 return lua.hook_get_revision_cert_trust(signers, id, name, val);
3787};
3788
3789// transaction guards
3790
3791conditional_transaction_guard::~conditional_transaction_guard()
3792{
3793 if (!acquired)
3794 return;
3795 if (committed)
3796 db.imp->commit_transaction();
3797 else
3798 db.imp->rollback_transaction();
3799}
3800
3801void
3802conditional_transaction_guard::acquire()
3803{
3804 I(!acquired);
3805 acquired = true;
3806 db.imp->begin_transaction(exclusive);
3807}
3808
3809void
3810conditional_transaction_guard::do_checkpoint()
3811{
3812 I(acquired);
3813 db.imp->commit_transaction();
3814 db.imp->begin_transaction(exclusive);
3815 checkpointed_calls = 0;
3816 checkpointed_bytes = 0;
3817}
3818
3819void
3820conditional_transaction_guard::maybe_checkpoint(size_t nbytes)
3821{
3822 I(acquired);
3823 checkpointed_calls += 1;
3824 checkpointed_bytes += nbytes;
3825 if (checkpointed_calls >= checkpoint_batch_size
3826 || checkpointed_bytes >= checkpoint_batch_bytes)
3827 do_checkpoint();
3828}
3829
3830void
3831conditional_transaction_guard::commit()
3832{
3833 I(acquired);
3834 committed = true;
3835}
3836
3837
3838
3839// Local Variables:
3840// mode: C++
3841// fill-column: 76
3842// c-file-style: "gnu"
3843// indent-tabs-mode: nil
3844// End:
3845// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status