Data formats

devel::bugtracker
devel::code-generator
devel::i18n
devel::lang:c
devel::lang:c++
implemented-in::c
implemented-in::c++
interface::3d
interface::commandline
interface::text-mode
works-with-format::xml
works-with-format::xml:rss
works-with-format::xml:xslt

# Empty field
Tag:
# Field with only one tag
Tag: web::server
# Field with many tags
Tag: made-of::data:font, role::app-data, use::printing, works-with::text, works-with-format::postscript
# Field with grouped tags
Tag: made-of::{data:font,data:html,data:pdf,data:tex}, role::app-data, role::documentation, suite::TODO, use::typesetting, works-with::{font,image,image:vector,text}, works-with-format::{pdf,postscript,tex}, x11::font

my ($packages, $tags) = split(/:\s+/, $line);
my @packages = split(/,\s+/, $packages);
my @tags = split(/,\s+/, $tags);

# Merge tag informations so that package names only appears once in the
# file
tagcoll -g copy my-tagdb

# Convert a packages -> tags mapping to a tags -> packages mapping
tagcoll reverse my-tagdb

# Compute the differences between two tag databases
tagcoll diff my-tagdb1 my-tagdb2 > tag-patch

# Apply a tag patch
tagcoll -p tag-patch copy my-tagdb1 > same-as-my-tagdb2

# Filter a tag database, keeping only the lines matching a given tag
# expression
tagcoll grep 'role::program && interface::x11' my-tagdb

xmame-x: hardware::emulation
abakus: field::mathematics, interface::x11, role::program, scope::utility, suite::kde, uitoolkit::qt, x11::application
gnomermind, gnotepad+, gnotime: interface::x11, role::program, scope::application, suite::gnome, uitoolkit::gtk, x11::application

Facet: protocol
Description: Network Protocol
Status: draft

Tag: protocol::TODO
Description: Need an extra tag
 The package can be categorised along this facet, but the right tag for it is
 missing.
 .
 Mark a package with this tag to signal the vocabulary maintainers of cases
 where the current tag set is lacking.

Tag: protocol::atm
Description: ATM
 Asynchronous Transfer Mode, a high speed protocol for communication between
 computers in a network.
 .
 While ATM is used to implement *DSL networks, it has never gained widespread
 use as a technology for building local area networks (LANs), for which it was
 originally intended.
 .
 Link: http://en.wikipedia.org/wiki/Asynchronous_Transfer_Mode

Tag: protocol::bittorrent
Description: BitTorrent
 BitTorrent is a protocol for peer-to-peer based file distribution over
 network.
 .
 Although the actual data transport happens between BitTorrent clients, one
 central node, the so-called trackers, is needed to keep a list of all clients
 that download or provide the same file.
 .
 Link: http://www.bittorrent.com/
 Link: http://en.wikipedia.org/wiki/BitTorrent

# Source locations to get tag database info
#
# You can specify more than one; during update, debtags will merge the contents
# of all the sources.  The merged data can be found in
# /var/lib/debtags/package-tags and /var/lib/debtags/vocabulary.

# Reviewed tag data from the APT Packages file
tags apt://

# Regularly updated, but unchecked, database on the web
# This tries to access the files:
#   http://debtags.alioth.debian.org/tags/tags-current.gz
#   http://debtags.alioth.debian.org/tags/vocabulary.gz
#
# The vocabulary, if existing, is used to filter the tag database: all
# tags in tags-current.gz are discarded if they are not found in
# the vocabulary.  After filtering, the vocabulary is merged with the
# other available vocabulary information.
#
# If the vocabulary is not found in the tag source, the tag data is
# filtered using the system vocabulary.
#
#tags http://debtags.alioth.debian.org/tags/

# Package rating from http://www.iterating.org
#tags http://www.iterating.org/tags

# Example of locally maintained custom tag data
#
# You can provide your own tag data.  To do so, place a file called
# vocabulary.gz and a file called tags-current.gz in a directory, and use the
# directory as a tag source.
#
# In the example given below, you would place the tag data in
# /etc/debtags/mytags/tags-current.gz, and the vocabulary data in
# /etc/debtags/mytags/vocabulary.gz
#
# The format of tags-current.gz is described in the tagcoll(1) manpage and the
# file is similar to /var/lib/debtags/package-tags.
#
# The format of vocabulary.gz is like the Debian Packages file, and the file is
# similar to /var/lib/debtags/vocabulary.
#
#tags file:/etc/debtags/mytags

[size of index 1][index1][size of index 2][index2]...

struct Item {
    // Offset of the facet record in the text vocabulary file
    int offset;
    // Size of the facet record in the text vocabulary file
    int size;
    // Tag index ID of the first tag in this facet
    int firsttag;
    // Tag index ID of the last tag in this facet
    int lasttag;
    // Facet name (0-terminated)
    const char name[];
};

struct Item {
    // Offset of the tag record in the text vocabulary file
    int offset;
    // Size of the tag record in the text vocabulary file
    int size;
    // Facet index ID of the facet that contains this tag
    int facet;
    // Tag name (0-terminated, includes the facet name)
    const char name[];
};

// Build the facet index
void VocabularyMerger::FacetIndexer::encode(char* buf) const
{
    int pos = vm.facets.size() * sizeof(int);

    for (std::map<std::string, FacetData>::const_iterator f = vm.facets.begin(); f != vm.facets.end(); f++)
    {
        ((int*)buf)[f->second.id] = pos;

        // offset of record in vocabulary
        *(int*)(buf+pos) = f->second.ofs;
        pos += sizeof(int);

        // size of record in vocabulary
        *(int*)(buf+pos) = f->second.len;
        pos += sizeof(int);

        // id of first tag
        *(int*)(buf+pos) = f->second.tags.begin()->second.id;
        pos += sizeof(int);

        // id of last tag
        *(int*)(buf+pos) = f->second.tags.rbegin()->second.id;
        pos += sizeof(int);

        // name (0-terminated)
        memcpy(buf + pos, f->first.c_str(), f->first.size() + 1);
        pos += f->first.size() + 1;

        // Align to int boundaries
        if ((pos % sizeof(int)) != 0)
            pos = (pos + sizeof(int)) / sizeof(int) * sizeof(int);
    }
}

// Build the tag index
void VocabularyMerger::TagIndexer::encode(char* buf) const
{
    int pos = vm.tagCount * sizeof(int);

    for (std::map<std::string, FacetData>::const_iterator f = vm.facets.begin(); f != vm.facets.end(); f++)
    {
        for (std::map<std::string, TagData>::const_iterator t = f->second.tags.begin();
                t != f->second.tags.end(); t++)
        {
            ((int*)buf)[t->second.id] = pos;

            // offset of record in vocabulary
            *(int*)(buf+pos) = t->second.ofs;
            pos += sizeof(int);

            // size of record in vocabulary
            *(int*)(buf+pos) = t->second.len;
            pos += sizeof(int);

            // id of facet
            *(int*)(buf+pos) = f->second.id;
            pos += sizeof(int);

            // name (0-terminated)
            string name = f->first + "::" + t->first;
            memcpy(buf + pos, name.c_str(), name.size() + 1);
            pos += name.size() + 1;

            // Align to int boundaries
            if ((pos % sizeof(int)) != 0)
                pos = (pos + sizeof(int)) / sizeof(int) * sizeof(int);
        }
    }
}

// Get the package name from the package ID
const char* name(int id) const
{
    if (id >= 0 || static_cast<unsigned>(id) < size())
        return m_buf + ((int*)m_buf)[id];
    return NULL;
}

// Build the package name index
void encode(char* buf) const
{
    typedef typename ept::t::cache::Package<C> Package;

    // First sort the packages by ID
    std::vector<Package> pkgs;
    pkgs.resize(m_pkgs.index().packageCount() + 1);
    for (typename Index::iterator i = m_pkgs.index().begin(); i != m_pkgs.index().end(); ++i)
        pkgs[i->ondiskId()] = *i;

    // Then write them out
    int pos = pkgs.size() * sizeof(int);
    int idx = 0;
    for (typename std::vector<Package>::const_iterator i = pkgs.begin(); i != pkgs.end(); ++i)
    {
        ((int*)buf)[idx++] = pos;
        memcpy(buf + pos, i->name( std::string( "invalid" ) ).c_str(),
           i->name( std::string( "invalid" ) ).size() + 1);
        pos += i->name( std::string( "invalid" ) ).size() + 1;
    }
}

[offset of mapping for item 0, offset of mapping for item 1...]
[size of array][sorted array of ints pointed by index 0]
[size of array][sorted array of ints pointed by index 1]
[size of array][sorted array of ints pointed by index 2]
[...]
[number of items in the mapping]

// Get the array of tag IDs associated to a package ID
// (or the array of package IDs associated to a tag ID, depending on
//  what index we are accessing)
const int* data(int val) const
{
    return (val >= 0 && (unsigned)val < size()) ? buf() + ofs(val) + 1 : 0;
}
// Get the size of an ID array
size_t size(int val) const
{
    return (val >= 0 && (unsigned)val < size()) ? buf()[ofs(val)] : 0;
}
// Get the number of mappings in the index
size_t size() const { return ofs(0); }

/**
 * Creates an on-disk index to use for IntIndex
 */
class IntIndexer : public MMapIndexer, public std::vector<std::set<int> >
{
public:
    /// Store the key->val mapping into the indexer
    void map(unsigned int key, int val)
    {
        if (size() <= key)
            resize(key + 1);
        (*this)[key].insert(val);
    }

    /// Return the size of the encoded index data
    virtual int encodedSize() const;

    /// Write the index data in the given buffer, which should be at least
    /// encodedSize bytes
    virtual void encode(char* buf) const
    {
        int* buf = (int*)cbuf;
        int pos = size();
        for (size_t i = 0; i < size(); i++)
        {
            buf[i] = pos;
            buf[pos++] = (*this)[i].size();
            for (set<int>::const_iterator j = (*this)[i].begin(); j != (*this)[i].end(); j++)
                buf[pos++] = *j;
        }
    }
};