/*===========================================================================
Copyright (c) 1998-2000, The Santa Cruz Operation
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
*Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
*Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
*Neither name of The Santa Cruz Operation nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS
IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT falseT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT falseT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
DAMAGE.
=========================================================================*/
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#if SHARE
# include <sys/types.h>
# include <sys/ipc.h>
# include <sys/shm.h>
# define ERR -1
#endif
#include "invlib.h"
#include "global.h"
#include <assert.h>
#define DEBUG 0 /* debugging code and realloc messages */
#define BLOCKSIZE 2 * BUFSIZ /* logical block size */
#define POSTINC 10000 /* posting buffer size increment */
#define SEP ' ' /* sorted posting field separator */
#define SETINC 100 /* posting set size increment */
#define STATS 0 /* print statistics */
#define SUPERINC 10000 /* super index size increment */
#define TERMMAX 512 /* term max size */
#define FMTVERSION 1 /* inverted index format version */
#define ZIPFSIZE 200 /* zipf curve size */
#if DEBUG
/* FIXME HBB 20010705: nowhere in the source is `invbreak' ever set to
* a value other than the (silent) initialization to zero. Pretty
* useless, that looks */
int invbreak;
#endif
static int boolready(void);
static int invnewterm(void);
static void invstep(INVCONTROL *invcntl);
static void invcannotalloc(unsigned n);
static void invcannotopen(char *file);
static void invcannotwrite(char *file);
#if STATS
int showzipf; /* show postings per term distribution */
#endif
static POSTING *item, *enditem, *item1 = NULL, *item2 = NULL;
static unsigned int setsize1, setsize2;
static long numitems, totterm, zerolong;
static char *indexfile, *postingfile;
static FILE *outfile, *fpost;
static size_t supersize = SUPERINC, supintsize;
static unsigned int numpost, numlogblk, amtused, nextpost;
static unsigned int lastinblk, numinvitems;
static POSTING *POST, *postptr;
static unsigned long *SUPINT, *supint, nextsupfing;
static char *SUPFING, *supfing;
static char thisterm[TERMMAX];
typedef union logicalblk {
long invblk[BLOCKSIZE / sizeof(long)];
char chrblk[BLOCKSIZE];
} t_logicalblk;
static t_logicalblk logicalblk;
#if DEBUG || STATS
static long totpost;
#endif
#if STATS
static int zipf[ZIPFSIZE + 1];
#endif
long invmake(char *invname, char *invpost, FILE *infile) {
unsigned char *s;
long num;
int i;
long fileindex = 0; /* initialze, to avoid warning */
unsigned postsize = POSTINC * sizeof(*POST);
unsigned long *intptr;
char line[TERMMAX];
long tlong;
PARAM param;
POSTING posting;
char temp[BLOCKSIZE];
#if STATS
int j;
unsigned maxtermlen = 0;
#endif
/* output file */
if((outfile = vpfopen(invname, "w+b")) == NULL) {
invcannotopen(invname);
return (0);
}
indexfile = invname;
fseek(outfile, BUFSIZ, SEEK_SET);
/* posting file */
if((fpost = vpfopen(invpost, "wb")) == NULL) {
invcannotopen(invpost);
return (0);
}
postingfile = invpost;
nextpost = 0;
/* get space for the postings list */
if((POST = malloc(postsize)) == NULL) {
invcannotalloc(postsize);
return (0);
}
postptr = POST;
/* get space for the superfinger (superindex) */
if((SUPFING = malloc(supersize)) == NULL) {
invcannotalloc(supersize);
return (0);
}
supfing = SUPFING;
/* FIXME HBB: magic number alert (40) */
supintsize = supersize / 40u;
/* also for the superfinger index */
if((SUPINT = malloc(supintsize * sizeof(*SUPINT))) == NULL) {
invcannotalloc(supintsize * sizeof(*SUPINT));
return (0);
}
supint = SUPINT;
supint++; /* leave first term open for a count */
/* initialize using an empty term */
strcpy(thisterm, "");
*supint++ = 0;
*supfing++ = ' ';
*supfing++ = '\0';
nextsupfing = 2;
#if DEBUG || STATS
totpost = 0L;
#endif
totterm = 0L;
numpost = 1;
/* set up as though a block had come and gone, i.e., set up for new block */
/* 3 longs needed for: numinvitems, next block, and previous block */
amtused = 3 * sizeof(long);
numinvitems = 0;
numlogblk = 0;
lastinblk = sizeof(t_logicalblk);
/* now loop as long as more to read (till eof) */
while(fgets(line, TERMMAX, infile) != NULL) {
#if DEBUG || STATS
++totpost;
#endif
s = strchr(line, SEP);
if(s != NULL) {
*s = '\0';
} else {
continue;
}
#if STATS
if((i = strlen(line)) > maxtermlen) { maxtermlen = i; }
#endif
#if DEBUG
printf("%ld: %s ", totpost, line);
fflush(stdout);
#endif
if(strcmp(thisterm, line) == 0) {
if((postptr + 10) > (POST + (postsize / sizeof(*POST)))) {
i = postptr - POST;
postsize += POSTINC * sizeof(*POST);
if((POST = realloc(POST, postsize)) == NULL) {
invcannotalloc(postsize);
return (0);
}
postptr = i + POST;
#if DEBUG
printf("reallocated post space to %u, totpost=%ld\n", postsize, totpost);
#endif
}
numpost++;
} else {
/* have a new term */
if(!invnewterm()) { return (0); }
strcpy(thisterm, line);
numpost = 1;
postptr = POST;
fileindex = 0;
}
/* get the new posting */
num = *++s - '!';
i = 1;
do {
num = BASE * num + *++s - '!';
} while(++i < PRECISION);
posting.lineoffset = num;
while(++fileindex < nsrcfiles && num > srcoffset[fileindex]) {
;
}
posting.fileindex = --fileindex;
posting.type = *++s;
++s;
if(*s != '\n') {
num = *++s - '!';
while(*++s != '\n') {
num = BASE * num + *s - '!';
}
posting.fcnoffset = num;
} else {
posting.fcnoffset = 0;
}
*postptr++ = posting;
#if DEBUG
printf("%ld %ld %ld %ld\n",
posting.fileindex,
posting.fcnoffset,
posting.lineoffset,
posting.type);
fflush(stdout);
#endif
}
if(!invnewterm()) { return (0); }
/* now clean up final block */
logicalblk.invblk[0] = numinvitems;
/* loops pointer around to start */
logicalblk.invblk[1] = 0;
logicalblk.invblk[2] = numlogblk - 1;
if(fwrite(&logicalblk, sizeof(t_logicalblk), 1, outfile) == 0) { goto cannotwrite; }
numlogblk++;
/* write out block to save space. what in it doesn't matter */
if(fwrite(&logicalblk, sizeof(t_logicalblk), 1, outfile) == 0) { goto cannotwrite; }
/* finish up the super finger */
*SUPINT = numlogblk;
/* add to the offsets the size of the offset pointers */
intptr = (SUPINT + 1);
i = (char *)supint - (char *)SUPINT;
while(intptr < supint)
*intptr++ += i;
/* write out the offsets (1 for the N at start) and the super finger */
if(fwrite(SUPINT, sizeof(*SUPINT), numlogblk + 1, outfile) == 0 ||
fwrite(SUPFING, 1, supfing - SUPFING, outfile) == 0) {
goto cannotwrite;
}
/* save the size for reference later */
nextsupfing = sizeof(long) + sizeof(long) * numlogblk + (supfing - SUPFING);
/* make sure the file ends at a logical block boundary. This is
necessary for invinsert to correctly create extended blocks
*/
i = nextsupfing % sizeof(t_logicalblk);
/* write out junk to fill log blk */
if(fwrite(temp, sizeof(t_logicalblk) - i, 1, outfile) == 0 ||
fflush(outfile) == EOF) { /* rewind doesn't check for write failure */
goto cannotwrite;
}
/* write the control area */
rewind(outfile);
param.version = FMTVERSION;
param.filestat = 0;
param.sizeblk = sizeof(t_logicalblk);
param.startbyte = (numlogblk + 1) * sizeof(t_logicalblk) + BUFSIZ;
;
param.supsize = nextsupfing;
param.cntlsize = BUFSIZ;
param.share = 0;
if(fwrite(¶m, sizeof(param), 1, outfile) == 0) { goto cannotwrite; }
for(i = 0; i < 10; i++) /* for future use */
if(fwrite(&zerolong, sizeof(zerolong), 1, outfile) == 0) { goto cannotwrite; }
/* make first block loop backwards to last block */
if(fflush(outfile) == EOF) { /* fseek doesn't check for write failure */
goto cannotwrite;
}
/* get to second word first block */
fseek(outfile, BUFSIZ + 2 * sizeof(long), SEEK_SET);
tlong = numlogblk - 1;
if(fwrite(&tlong, sizeof(tlong), 1, outfile) == 0 || fclose(outfile) == EOF) {
cannotwrite:
invcannotwrite(invname);
return (0);
}
if(fclose(fpost) == EOF) {
invcannotwrite(postingfile);
return (0);
}
--totterm; /* don't count null term */
#if STATS
printf("logical blocks = %d, postings = %ld, terms = %ld, max term length = %d\n",
numlogblk,
totpost,
totterm,
maxtermlen);
if(showzipf) {
printf("\n************* ZIPF curve ****************\n");
for(j = ZIPFSIZE; j > 1; j--)
if(zipf[j]) break;
for(i = 1; i < j; ++i) {
printf("%3d -%6d ", i, zipf[i]);
if(i % 6 == 0) putchar('\n');
}
printf(">%d-%6d\n", ZIPFSIZE, zipf[0]);
}
#endif
/* free all malloc'd memory */
free(POST);
free(SUPFING);
free(SUPINT);
return (totterm);
}
/* add a term to the data base */
static int invnewterm(void) {
int backupflag, i, j, holditems, gooditems, howfar;
unsigned int maxback, len, numwilluse, wdlen;
char *tptr, *tptr3;
union {
unsigned long packword[2];
ENTRY e;
} iteminfo;
gooditems = 0; /* initialize, to avoid warning */
totterm++;
#if STATS
/* keep zipfian info on the distribution */
if(numpost <= ZIPFSIZE)
zipf[numpost]++;
else
zipf[0]++;
#endif
len = strlen(thisterm);
/* length of term rounded up to long boundary */
wdlen = (len + (sizeof(long) - 1)) / sizeof(long);
/* each term needs 2 longs for its iteminfo and
* 1 long for its offset */
numwilluse = (wdlen + 3) * sizeof(long);
/* new block if at least 1 item in block */
if(numinvitems && numwilluse + amtused > sizeof(t_logicalblk)) {
/* set up new block */
if(supfing + 500u > SUPFING + supersize) {
i = supfing - SUPFING;
supersize += 20000u;
if((SUPFING = realloc(SUPFING, supersize)) == NULL) {
invcannotalloc(supersize);
return (0);
}
supfing = i + SUPFING;
#if DEBUG
printf("reallocated superfinger space to %d, totpost=%ld\n",
supersize,
totpost);
#endif
}
/* check that room for the offset as well */
/* FIXME HBB: magic number alert (10) */
if((numlogblk + 10) > supintsize) {
i = supint - SUPINT;
supintsize += SUPERINC;
if((SUPINT = realloc(SUPINT, supintsize * sizeof(*SUPINT))) == NULL) {
invcannotalloc(supintsize * sizeof(*SUPINT));
return (0);
}
supint = i + SUPINT;
#if DEBUG
printf("reallocated superfinger offset to %d, totpost = %ld\n",
supintsize * sizeof(*SUPINT),
totpost);
#endif
}
/* See if backup is efficatious */
backupflag = 0;
maxback = (int)strlen(thisterm) / 10;
holditems = numinvitems;
if(maxback > numinvitems) maxback = numinvitems - 2;
howfar = 0;
while(maxback-- > 1) {
howfar++;
iteminfo.packword[0] =
logicalblk.invblk[--holditems * 2 + (sizeof(long) - 1)];
if((i = iteminfo.e.size / 10) < maxback) {
maxback = i;
backupflag = howfar;
gooditems = holditems;
}
}
/* see if backup will occur */
if(backupflag) { numinvitems = gooditems; }
logicalblk.invblk[0] = numinvitems;
/* set forward pointer pointing to next */
logicalblk.invblk[1] = numlogblk + 1;
/* set back pointer to last block */
logicalblk.invblk[2] = numlogblk - 1;
if(fwrite(logicalblk.chrblk, 1, sizeof(t_logicalblk), outfile) == 0) {
invcannotwrite(indexfile);
return (0);
}
/* 3 longs needed for: numinvitems, next block, and previous block */
amtused = 3 * sizeof(long);
numlogblk++;
/* check if had to back up, if so do it */
if(backupflag) {
char *tptr2;
/* find out where the end of the new block is */
iteminfo.packword[0] = logicalblk.invblk[numinvitems * 2 + 1];
tptr3 = logicalblk.chrblk + iteminfo.e.offset;
/* move the index for this block */
for(i = 3; i <= (backupflag * 2 + 2); i++)
logicalblk.invblk[i] = logicalblk.invblk[numinvitems * 2 + i];
/* move the word into the super index */
iteminfo.packword[0] = logicalblk.invblk[3];
iteminfo.packword[1] = logicalblk.invblk[4];
tptr2 = logicalblk.chrblk + iteminfo.e.offset;
strncpy(supfing, tptr2, (int)iteminfo.e.size);
*(supfing + iteminfo.e.size) = '\0';
#if DEBUG
printf("backup %d at term=%s to term=%s\n", backupflag, thisterm, supfing);
#endif
*supint++ = nextsupfing;
nextsupfing += strlen(supfing) + 1;
supfing += strlen(supfing) + 1;
/* now fix up the logical block */
tptr = logicalblk.chrblk + lastinblk;
lastinblk = sizeof(t_logicalblk);
tptr2 = logicalblk.chrblk + lastinblk;
j = tptr3 - tptr;
while(tptr3 > tptr)
*--tptr2 = *--tptr3;
lastinblk -= j;
amtused += ((2 * sizeof(long)) * backupflag + j);
for(i = 3; i < (backupflag * 2 + 2); i += 2) {
iteminfo.packword[0] = logicalblk.invblk[i];
iteminfo.e.offset += (tptr2 - tptr3);
logicalblk.invblk[i] = iteminfo.packword[0];
}
numinvitems = backupflag;
} else { /* no backup needed */
numinvitems = 0;
lastinblk = sizeof(t_logicalblk);
/* add new term to superindex */
strcpy(supfing, thisterm);
supfing += strlen(thisterm) + 1;
*supint++ = nextsupfing;
nextsupfing += strlen(thisterm) + 1;
}
}
/* HBB 20010501: Fixed bug by replacing magic number '8' by
* what it actually represents. */
lastinblk -= (numwilluse - 2 * sizeof(long));
iteminfo.e.offset = lastinblk;
iteminfo.e.size = len;
iteminfo.e.space = 0;
iteminfo.e.post = numpost;
strncpy(logicalblk.chrblk + lastinblk, thisterm, len);
amtused += numwilluse;
logicalblk.invblk[(lastinblk / sizeof(long)) + wdlen] = nextpost;
if((i = postptr - POST) > 0) {
if(fwrite(POST, sizeof(*POST), i, fpost) == 0) {
invcannotwrite(postingfile);
return (0);
}
nextpost += i * sizeof(*POST);
}
logicalblk.invblk[3 + 2 * numinvitems++] = iteminfo.packword[0];
logicalblk.invblk[2 + 2 * numinvitems] = iteminfo.packword[1];
return (1);
}
/*
* If 'invname' ends with the 'from' substring, it is replaced inline with the
* 'to' substring (which must be of the exact same length), and the function
* returns 0. Otherwise, returns -1.
*/
static int invflipname(char *invname, const char *from, const char *to) {
char *temp, *i = NULL;
assert(strlen(from) == strlen(to));
temp = invname - 1;
while((temp = strstr(temp + 1, from)))
i = temp;
if(!i || i[strlen(from)] != '\0') return -1;
while(*to)
*i++ = *to++;
return 0;
}
/* small helper function to centralize handling of binary opening
* for reading, and use of the 'stat" flag */
static FILE *open_for_reading(char *name, int stat) {
return vpfopen(name, ((stat == 0) ? "rb" : "r+b"));
}
/* handle opening of a file under a possibly "flipped" name */
/* If db created without '-f', but now invoked with '-f cscope.out',
* we need to check for 'cscope.in.out', rather than 'cscope.out.in':
* I.e, hack around our own violation of the inverse db naming convention */
/* more silliness: if you create the db with '-f cscope', then try to open
* it without '-f cscope', you'll fail unless we check for 'cscope.out.in'
* here. */
static FILE *open_file_with_flipped_name(char *name, const char *flip_in,
const char *flip_out, int stat) {
if(!invflipname(name, flip_in, flip_out)) {
FILE *fptr = open_for_reading(name, stat);
if(!fptr) /* flip back for error message */
invflipname(name, flip_out, flip_in);
return fptr;
};
return 0;
}
static FILE *open_file_with_possibly_flipped_name(char *name, const char *flip1,
const char *flip2, int stat) {
FILE *fptr = open_for_reading(name, stat);
if(!fptr) fptr = open_file_with_flipped_name(name, flip2, flip1, stat);
if(!fptr) fptr = open_file_with_flipped_name(name, flip1, flip2, stat);
return fptr;
}
int invopen(INVCONTROL *invcntl, char *invname, char *invpost, int stat) {
int read_index;
invcntl->invfile =
open_file_with_possibly_flipped_name(invname, INVNAME, INVNAME2, stat);
if(!invcntl->invfile) {
invcannotopen(invname);
return (-1);
}
if(fread(&invcntl->param, sizeof(invcntl->param), 1, invcntl->invfile) == 0) {
fprintf(stderr, PROGRAM_NAME ": empty inverted file\n");
fclose(invcntl->invfile);
return (-1);
}
if(invcntl->param.version != FMTVERSION) {
fprintf(stderr,
PROGRAM_NAME
": cannot read old index format; use -U option to force database to rebuild\n");
fclose(invcntl->invfile);
return (-1);
}
assert(invcntl->param.sizeblk == sizeof(t_logicalblk));
if(stat == 0 && invcntl->param.filestat == INVALONE) {
fprintf(stderr, PROGRAM_NAME ": inverted file is locked\n");
fclose(invcntl->invfile);
return (-1);
}
invcntl->postfile =
open_file_with_possibly_flipped_name(invpost, INVPOST, INVPOST2, stat);
if(!invcntl->postfile) {
invcannotopen(invpost);
fclose(invcntl->invfile);
return (-1);
}
/* allocate core for a logical block */
if((invcntl->logblk = malloc((size_t)invcntl->param.sizeblk)) == NULL) {
invcannotalloc((size_t)invcntl->param.sizeblk);
fclose(invcntl->postfile);
fclose(invcntl->invfile);
return (-1);
}
/* allocate for and read in superfinger */
read_index = 1;
invcntl->iindex = NULL;
#if SHARE
if(invcntl->param.share == 1) {
key_t shm_key;
struct shmid_ds shm_buf;
int shm_id;
/* see if the shared segment exists */
shm_key = ftok(invname, 2);
shm_id = shmget(shm_key, 0, 0);
/* Failure simply means (hopefully) that segment doesn't exists */
if(shm_id == -1) {
/* Have to give general write permission due to AMdahl not having protected
* segments */
shm_id =
shmget(shm_key, invcntl->param.supsize + sizeof(long), IPC_CREAT | 0666);
if(shm_id == -1) perror("Could not create shared memory segment");
} else
read_index = 0;
if(shm_id != -1) {
invcntl->iindex = shmat(shm_id, 0, ((read_index) ? 0 : SHM_RDONLY));
if(invcntl->iindex == (char *)ERR) {
fprintf(stderr, PROGRAM_NAME ": shared memory link failed\n");
invcntl->iindex = NULL;
read_index = 1;
}
}
}
#endif
if(invcntl->iindex == NULL) /* FIXME HBB: magic number alert (4, sizeof(long)) */
invcntl->iindex = malloc((size_t)invcntl->param.supsize + 4 * sizeof(long));
if(invcntl->iindex == NULL) {
invcannotalloc((size_t)invcntl->param.supsize);
free(invcntl->logblk);
fclose(invcntl->postfile);
fclose(invcntl->invfile);
return (-1);
}
if(read_index) {
fseek(invcntl->invfile, invcntl->param.startbyte, SEEK_SET);
fread(invcntl->iindex, (int)invcntl->param.supsize, 1, invcntl->invfile);
}
invcntl->numblk = -1;
if(boolready() == -1) {
fclose(invcntl->postfile);
fclose(invcntl->invfile);
return (-1);
}
/* write back out the control block if anything changed */
invcntl->param.filestat = stat;
if(stat > invcntl->param.filestat) {
rewind(invcntl->invfile);
fwrite(&invcntl->param, sizeof(invcntl->param), 1, invcntl->invfile);
}
return (1);
}
/** invclose must be called to wrap things up and deallocate core **/
void invclose(INVCONTROL *invcntl) {
/* write out the control block in case anything changed */
if(invcntl->param.filestat > 0) {
invcntl->param.filestat = 0;
rewind(invcntl->invfile);
fwrite(&invcntl->param, 1, sizeof(invcntl->param), invcntl->invfile);
}
if(invcntl->param.filestat == INVALONE) {
/* write out the super finger */
fseek(invcntl->invfile, invcntl->param.startbyte, SEEK_SET);
fwrite(invcntl->iindex, 1, (int)invcntl->param.supsize, invcntl->invfile);
}
fclose(invcntl->invfile);
fclose(invcntl->postfile);
#if SHARE
if(invcntl->param.share > 0) {
shmdt(invcntl->iindex);
invcntl->iindex = NULL;
}
#endif
if(invcntl->iindex != NULL) free(invcntl->iindex);
free(invcntl->logblk);
}
/** invstep steps the inverted file forward one item **/
static void invstep(INVCONTROL *invcntl) {
if(invcntl->keypnt < (invcntl->logblk->invblk[0] - 1)) {
invcntl->keypnt++;
return;
}
/* move forward a block else wrap */
invcntl->numblk =
invcntl->logblk->invblk[1]; /* was: *(int *)(invcntl->logblk + sizeof(long))*/
/* now read in the block */
fseek(invcntl->invfile,
invcntl->numblk * invcntl->param.sizeblk + invcntl->param.cntlsize,
SEEK_SET);
fread(invcntl->logblk, (int)invcntl->param.sizeblk, 1, invcntl->invfile);
invcntl->keypnt = 0;
}
/** invforward moves forward one term in the inverted file **/
int invforward(INVCONTROL *invcntl) {
invstep(invcntl);
/* skip things with 0 postings */
/* FIXME HBB: magic number alert! (3) */
while(((ENTRY *)(invcntl->logblk->invblk + 3) + invcntl->keypnt)->post == 0) {
invstep(invcntl);
}
/* Check for having wrapped - reached start of inverted file! */
if((invcntl->numblk == 0) && (invcntl->keypnt == 0)) return (0);
return (1);
}
/** invterm gets the present term from the present logical block **/
long invterm(INVCONTROL *invcntl, char *term) {
ENTRY *entryptr;
/* FIXME HBB: magic number alert! (3) */
entryptr = (ENTRY *)(invcntl->logblk->invblk + 3) + invcntl->keypnt;
strncpy(term, invcntl->logblk->chrblk + entryptr->offset, (int)entryptr->size);
*(term + entryptr->size) = '\0';
return (entryptr->post);
}
/** invfind searches for an individual item in the inverted file **/
long invfind(INVCONTROL *invcntl, char *searchterm) /* term being searched for */
{
int imid, ilow, ihigh;
long num;
int i;
unsigned long *intptr, *intptr2;
ENTRY *entryptr;
/* make sure it is initialized via invready */
if(invcntl->invfile == 0) return (-1L);
/* now search for the appropriate finger block */
intptr = (unsigned long *)invcntl->iindex;
ilow = 0;
ihigh = *intptr++ - 1;
while(ilow <= ihigh) {
imid = (ilow + ihigh) / 2;
intptr2 = intptr + imid;
i = strcmp(searchterm, (invcntl->iindex + *intptr2));
if(i < 0)
ihigh = imid - 1;
else if(i > 0)
ilow = ++imid;
else {
ilow = imid + 1;
break;
}
}
/* be careful about case where searchterm is after last in this block */
imid = (ilow) ? ilow - 1 : 0;
/* fetch the appropriate logical block if not in core */
/* note always fetch it if the file is busy */
if((imid != invcntl->numblk) || (invcntl->param.filestat >= INVBUSY)) {
fseek(invcntl->invfile,
(imid * invcntl->param.sizeblk) + invcntl->param.cntlsize,
SEEK_SET);
invcntl->numblk = imid;
fread(invcntl->logblk, (int)invcntl->param.sizeblk, 1, invcntl->invfile);
}
srch_ext:
/* now find the term in this block. tricky this */
intptr = (unsigned long *)invcntl->logblk->invblk;
ilow = 0;
ihigh = *intptr - 1;
intptr += 3;
num = 0;
while(ilow <= ihigh) {
imid = (ilow + ihigh) / 2;
entryptr = (ENTRY *)intptr + imid;
i = strncmp(searchterm,
invcntl->logblk->chrblk + entryptr->offset,
(int)entryptr->size);
if(i == 0) i = strlen(searchterm) - entryptr->size;
if(i < 0)
ihigh = imid - 1;
else if(i > 0)
ilow = ++imid;
else {
num = entryptr->post;
break;
}
}
/* be careful about case where searchterm is after last in this block */
if(imid >= invcntl->logblk->invblk[0]) {
invcntl->keypnt = invcntl->logblk->invblk[0];
invstep(invcntl);
/* note if this happens the term could be in extended block */
if(invcntl->param.startbyte < invcntl->numblk * invcntl->param.sizeblk)
goto srch_ext;
} else
invcntl->keypnt = imid;
return (num);
}
#if DEBUG
/** invdump dumps the block the term parameter is in **/
void invdump(INVCONTROL *invcntl, char *term) {
long i, j, n, *longptr;
ENTRY *entryptr;
char temp[512], *ptr;
/* dump superindex if term is "-" */
if(*term == '-') {
j = atoi(term + 1);
longptr = (long *)invcntl->iindex;
n = *longptr++;
printf("Superindex dump, num blocks=%ld\n", n);
longptr += j;
while((longptr <= ((long *)invcntl->iindex) + n) && invbreak == 0) {
printf("%2ld %6ld %s\n", j++, *longptr, invcntl->iindex + *longptr);
longptr++;
}
return;
} else if(*term == '#') {
j = atoi(term + 1);
/* fetch the appropriate logical block */
invcntl->numblk = j;
fseek(invcntl->invfile,
(j * invcntl->param.sizeblk) + invcntl->param.cntlsize,
SEEK_SET);
fread(invcntl->logblk, (int)invcntl->param.sizeblk, 1, invcntl->invfile);
} else
i = abs((int)invfind(invcntl, term));
longptr = invcntl->logblk->invblk;
n = *longptr++;
printf("Entry term to invdump=%s, postings=%ld, forwrd ptr=%ld, back ptr=%ld\n",
term,
i,
*(longptr),
*(longptr + 1));
/* FIXME HBB: magic number alert! (3) */
entryptr = (ENTRY *)(invcntl->logblk->invblk + 3);
printf("%ld terms in this block, block=%ld\n", n, invcntl->numblk);
printf("\tterm\t\t\tposts\tsize\toffset\tspace\t1st word\n");
for(j = 0; j < n && invbreak == 0; j++) {
ptr = invcntl->logblk->chrblk + entryptr->offset;
strncpy(temp, ptr, (int)entryptr->size);
temp[entryptr->size] = '\0';
ptr +=
(sizeof(long) * (long)((entryptr->size + (sizeof(long) - 1)) / sizeof(long)));
printf("%2ld %-24s\t%5ld\t%3d\t%d\t%d\t%ld\n",
j,
temp,
entryptr->post,
entryptr->size,
entryptr->offset,
entryptr->space,
*(long *)ptr);
entryptr++;
}
}
#endif
static int boolready(void) {
numitems = 0;
if(item1 != NULL) free(item1);
setsize1 = SETINC;
if((item1 = malloc(SETINC * sizeof(*item1))) == NULL) {
invcannotalloc(SETINC);
return (-1);
}
if(item2 != NULL) free(item2);
setsize2 = SETINC;
if((item2 = malloc(SETINC * sizeof(*item2))) == NULL) {
invcannotalloc(SETINC);
return (-1);
}
item = item1;
enditem = item;
return (0);
}
void boolclear(void) {
numitems = 0;
item = item1;
enditem = item;
}
POSTING *boolfile(INVCONTROL *invcntl, long *num, int boolarg) {
ENTRY *entryptr;
FILE *file;
void *ptr;
unsigned long *ptr2;
POSTING *newitem = NULL; /* initialize, to avoid warning */
POSTING posting;
unsigned u;
POSTING *newsetp = NULL, *set1p;
long newsetc, set1c, set2c;
/* FIXME HBB: magic number alert! (3) */
entryptr = (ENTRY *)(invcntl->logblk->invblk + 3) + invcntl->keypnt;
ptr = invcntl->logblk->chrblk + entryptr->offset;
ptr2 = ((unsigned long *)ptr) + (entryptr->size + (sizeof(long) - 1)) / sizeof(long);
*num = entryptr->post;
switch(boolarg) {
case bool_OR:
case falseT:
if(*num == 0) {
*num = numitems;
return (item);
}
}
/* make room for the new set */
u = 0;
switch(boolarg) {
case AND:
case falseT:
newsetp = item;
break;
case bool_OR:
u = enditem - item;
/* FALLTHROUGH */
case REVERSEfalseT:
u += *num;
if(item == item2) {
if(u > setsize1) {
u += SETINC;
if((item1 = realloc(item1, u * sizeof(*item1))) == NULL) {
invcannotalloc(u * sizeof(*item1));
boolready();
*num = -1;
return (NULL);
}
setsize1 = u;
}
newitem = item1;
} else {
if(u > setsize2) {
u += SETINC;
if((item2 = realloc(item2, u * sizeof(*item2))) == NULL) {
invcannotalloc(u * sizeof(*item2));
boolready();
*num = -1;
return (NULL);
}
setsize2 = u;
}
newitem = item2;
}
newsetp = newitem;
}
file = invcntl->postfile;
fseek(file, *ptr2, SEEK_SET);
fread(&posting, sizeof(posting), 1, file);
newsetc = 0;
switch(boolarg) {
case bool_OR:
/* while something in both sets */
set1p = item;
newsetp = newitem;
for(set1c = 0, set2c = 0; set1c < numitems && set2c < *num; newsetc++) {
if(set1p->lineoffset < posting.lineoffset) {
*newsetp++ = *set1p++;
set1c++;
} else if(set1p->lineoffset > posting.lineoffset) {
*newsetp++ = posting;
fread(&posting, (int)sizeof(posting), 1, file);
set2c++;
} else if(set1p->type < posting.type) {
*newsetp++ = *set1p++;
set1c++;
} else if(set1p->type > posting.type) {
*newsetp++ = posting;
fread(&posting, (int)sizeof(posting), 1, file);
set2c++;
} else { /* identical postings */
*newsetp++ = *set1p++;
set1c++;
fread(&posting, (int)sizeof(posting), 1, file);
set2c++;
}
}
/* find out what ran out and move the rest in */
if(set1c < numitems) {
newsetc += numitems - set1c;
while(set1c++ < numitems) {
*newsetp++ = *set1p++;
}
} else {
while(set2c++ < *num) {
*newsetp++ = posting;
newsetc++;
fread(&posting, (int)sizeof(posting), 1, file);
}
}
item = newitem;
break; /* end of bool_OR */
}
numitems = newsetc;
*num = newsetc;
enditem = (POSTING *)newsetp;
return ((POSTING *)item);
}
static void invcannotalloc(unsigned n) {
fprintf(stderr, PROGRAM_NAME ": cannot allocate %u bytes\n", n);
}
static void invcannotopen(char *file) {
fprintf(stderr, PROGRAM_NAME ": cannot open file %s\n", file);
}
static void invcannotwrite(char *file) {
perror(PROGRAM_NAME); /* must be first to preserve errno */
fprintf(stderr, PROGRAM_NAME ": write to file %s failed\n", file);
}