opensourcemirror
/
postgresql
mirror of git://git.postgresql.org/git/postgresql.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Eliminate query length limitation imposed by pg_client_to_server 

and pg_server_to_client.  Eliminate copy.c's restriction on the length
of a single attribute.
This commit is contained in:
Tom Lane 1999-09-11 22:28:11 +00:00
parent b65ab31910
commit 4644fc8071
4 changed files with 138 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
src/backend/commands/copy.c
View File

@ -6,7 +6,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.86 1999/07/22 02:40:06 tgl Exp $
* $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.87 1999/09/11 22:28:11 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -26,6 +26,7 @@
#include "commands/copy.h"
#include "commands/trigger.h"
#include "executor/executor.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "miscadmin.h"
#include "utils/acl.h"
@ -51,14 +52,10 @@ static void GetIndexRelations(Oid main_relation_oid,
int *n_indices,
Relation **index_rels);
#ifdef COPY_PATCH
static void CopyReadNewline(FILE *fp, int *newline);
static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline);
#else
static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim);
#endif
static void CopyAttributeOut(FILE *fp, char *string, char *delim, int is_array);
static void CopyAttributeOut(FILE *fp, char *string, char *delim);
static int CountTuples(Relation relation);
/*
@ -431,7 +428,7 @@ CopyTo(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
{
string = (char *) (*fmgr_faddr(&out_functions[i]))
(value, elements[i], typmod[i]);
CopyAttributeOut(fp, string, delim, attr[i]->attnelems);
CopyAttributeOut(fp, string, delim);
pfree(string);
}
else
@ -691,18 +688,12 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
{
if (!binary)
{
#ifdef COPY_PATCH
int newline = 0;
#endif
lineno++;
if (oids)
{
#ifdef COPY_PATCH
string = CopyReadAttribute(fp, &isnull, delim, &newline);
#else
string = CopyReadAttribute(fp, &isnull, delim);
#endif
if (string == NULL)
done = 1;
else
@ -710,19 +701,18 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
loaded_oid = oidin(string);
if (loaded_oid < BootstrapObjectIdData)
elog(ERROR, "COPY TEXT: Invalid Oid. line: %d", lineno);
pfree(string);
}
}
for (i = 0; i < attr_count && !done; i++)
{
#ifdef COPY_PATCH
string = CopyReadAttribute(fp, &isnull, delim, &newline);
#else
string = CopyReadAttribute(fp, &isnull, delim);
#endif
if (isnull)
{
values[i] = PointerGetDatum(NULL);
nulls[i] = 'n';
if (string)
pfree(string);
}
else if (string == NULL)
done = 1;
@ -739,12 +729,11 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
if (!PointerIsValid(values[i]) &&
!(rel->rd_att->attrs[i]->attbyval))
elog(ERROR, "copy from line %d: Bad file format", lineno);
pfree(string);
}
}
#ifdef COPY_PATCH
if (!done)
CopyReadNewline(fp, &newline);
#endif
}
else
{ /* binary */
@ -812,11 +801,6 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
if (done)
continue;
/*
* Does it have any sence ? - vadim 12/14/96
*
* tupDesc = CreateTupleDesc(attr_count, attr);
*/
tuple = heap_formtuple(tupDesc, values, nulls);
if (oids)
tuple->t_data->t_oid = loaded_oid;
@ -1086,30 +1070,18 @@ GetIndexRelations(Oid main_relation_oid,
}
}
#define EXT_ATTLEN (5 * BLCKSZ)
/*
returns 1 is c is in s
returns 1 if c is in s
*/
static bool
inString(char c, char *s)
{
int i;
if (s)
{
i = 0;
while (s[i] != '\0')
{
if (s[i] == c)
return 1;
i++;
}
}
if (s && c)
return strchr(s, c) != NULL;
return 0;
}
#ifdef COPY_PATCH
/*
* Reads input from fp until an end of line is seen.
*/
@ -1125,64 +1097,57 @@ CopyReadNewline(FILE *fp, int *newline)
*newline = 0;
}
#endif
/*
* Reads input from fp until eof is seen. If we are reading from standard
* input, AND we see a dot on a line by itself (a dot followed immediately
* by a newline), we exit as if we saw eof. This is so that copy pipelines
* can be used as standard input.
* Read the value of a single attribute.
*
* Result is either a palloc'd string, or NULL (if EOF or a null attribute).
* *isnull is set true if a null attribute, else false.
*
* delim is the string of acceptable delimiter characters(s).
* *newline remembers whether we've seen a newline ending this tuple.
*/
static char *
#ifdef COPY_PATCH
CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline)
#else
CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
#endif
{
static char attribute[EXT_ATTLEN];
StringInfoData attribute_buf;
char c;
int done = 0;
int i = 0;
#ifdef MULTIBYTE
int mblen;
int encoding;
unsigned char s[2];
char *cvt;
int j;
#endif
#ifdef MULTIBYTE
encoding = pg_get_client_encoding();
s[1] = 0;
#endif
#ifdef COPY_PATCH
/* if last delimiter was a newline return a NULL attribute */
if (*newline)
{
*isnull = (bool) true;
return NULL;
}
#endif
*isnull = (bool) false; /* set default */
if (CopyGetEof(fp))
return NULL;
while (!done)
initStringInfo(&attribute_buf);
if (CopyGetEof(fp))
goto endOfFile;
for (;;)
{
c = CopyGetChar(fp);
if (CopyGetEof(fp))
return NULL;
else if (c == '\\')
goto endOfFile;
if (c == '\\')
{
c = CopyGetChar(fp);
if (CopyGetEof(fp))
return NULL;
goto endOfFile;
switch (c)
{
case '0':
@ -1212,14 +1177,14 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
else
{
if (CopyGetEof(fp))
return NULL;
goto endOfFile;
CopyDonePeek(fp, c, 0); /* Return to stream! */
}
}
else
{
if (CopyGetEof(fp))
return NULL;
goto endOfFile;
CopyDonePeek(fp, c, 0); /* Return to stream! */
}
c = val & 0377;
@ -1244,66 +1209,70 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
c = '\v';
break;
case 'N':
attribute[0] = '\0'; /* just to be safe */
*isnull = (bool) true;
break;
case '.':
c = CopyGetChar(fp);
if (c != '\n')
elog(ERROR, "CopyReadAttribute - end of record marker corrupted. line: %d", lineno);
return NULL;
goto endOfFile;
break;
}
}
else if (inString(c, delim) || c == '\n')
else if (c == '\n' || inString(c, delim))
{
#ifdef COPY_PATCH
if (c == '\n')
*newline = 1;
#endif
done = 1;
break;
}
if (!done)
attribute[i++] = c;
appendStringInfoChar(&attribute_buf, c);
#ifdef MULTIBYTE
/* get additional bytes of the char, if any */
s[0] = c;
mblen = pg_encoding_mblen(encoding, s);
mblen--;
for (j = 0; j < mblen; j++)
for (j = 1; j < mblen; j++)
{
c = CopyGetChar(fp);
if (CopyGetEof(fp))
return NULL;
attribute[i++] = c;
goto endOfFile;
appendStringInfoChar(&attribute_buf, c);
}
#endif
if (i == EXT_ATTLEN - 1)
elog(ERROR, "CopyReadAttribute - attribute length too long. line: %d", lineno);
}
attribute[i] = '\0';
#ifdef MULTIBYTE
return (pg_client_to_server((unsigned char *) attribute, strlen(attribute)));
#else
return &attribute[0];
cvt = (char *) pg_client_to_server((unsigned char *) attribute_buf.data,
attribute_buf.len);
if (cvt != attribute_buf.data)
{
pfree(attribute_buf.data);
return cvt;
}
#endif
return attribute_buf.data;
endOfFile:
pfree(attribute_buf.data);
return NULL;
}
static void
CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
CopyAttributeOut(FILE *fp, char *server_string, char *delim)
{
char *string;
char c;
#ifdef MULTIBYTE
int mblen;
char *string_start;
int encoding;
int mblen;
int i;
#endif
#ifdef MULTIBYTE
string = pg_server_to_client(server_string, strlen(server_string));
encoding = pg_get_client_encoding();
string = (char *) pg_server_to_client((unsigned char *) server_string,
strlen(server_string));
string_start = string;
#else
string = server_string;
#endif
@ -1315,33 +1284,20 @@ CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
for (; (c = *string) != '\0'; string++)
#endif
{
if (c == delim[0] || c == '\n' ||
(c == '\\' && !is_array))
if (c == delim[0] || c == '\n' || c == '\\')
CopySendChar('\\', fp);
else if (c == '\\' && is_array)
{
if (*(string + 1) == '\\')
{
/* translate \\ to \\\\ */
CopySendChar('\\', fp);
CopySendChar('\\', fp);
CopySendChar('\\', fp);
string++;
}
else if (*(string + 1) == '"')
{
/* translate \" to \\\" */
CopySendChar('\\', fp);
CopySendChar('\\', fp);
}
}
#ifdef MULTIBYTE
for (i = 0; i < mblen; i++)
CopySendChar(*(string + i), fp);
#else
CopySendChar(*string, fp);
CopySendChar(c, fp);
#endif
}
#ifdef MULTIBYTE
if (string_start != server_string)
pfree(string_start); /* pfree pg_server_to_client result */
#endif
}
/*

41
src/backend/libpq/pqformat.c
View File

@ -15,7 +15,7 @@
*
* Copyright (c) 1994, Regents of the University of California
*
* $Id: pqformat.c,v 1.8 1999/08/31 04:26:37 tgl Exp $
* $Id: pqformat.c,v 1.9 1999/09/11 22:28:05 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -125,13 +125,16 @@ void
pq_sendcountedtext(StringInfo buf, const char *str, int slen)
{
#ifdef MULTIBYTE
const char *p;
char *p;
p = (const char *) pg_server_to_client((unsigned char *) str, slen);
p = (char *) pg_server_to_client((unsigned char *) str, slen);
if (p != str) /* actual conversion has been done? */
{
str = p;
slen = strlen(str);
slen = strlen(p);
pq_sendint(buf, slen + 4, 4);
appendBinaryStringInfo(buf, p, slen);
pfree(p);
return;
}
#endif
pq_sendint(buf, slen + 4, 4);
@ -149,15 +152,16 @@ void
pq_sendstring(StringInfo buf, const char *str)
{
int slen = strlen(str);
#ifdef MULTIBYTE
const char *p;
char *p;
p = (const char *) pg_server_to_client((unsigned char *) str, slen);
p = (char *) pg_server_to_client((unsigned char *) str, slen);
if (p != str) /* actual conversion has been done? */
{
str = p;
slen = strlen(str);
slen = strlen(p);
appendBinaryStringInfo(buf, p, slen + 1);
pfree(p);
return;
}
#endif
appendBinaryStringInfo(buf, str, slen + 1);
@ -229,15 +233,15 @@ int
pq_puttextmessage(char msgtype, const char *str)
{
int slen = strlen(str);
#ifdef MULTIBYTE
const char *p;
char *p;
p = (const char *) pg_server_to_client((unsigned char *) str, slen);
p = (char *) pg_server_to_client((unsigned char *) str, slen);
if (p != str) /* actual conversion has been done? */
{
str = p;
slen = strlen(str);
int result = pq_putmessage(msgtype, p, strlen(p) + 1);
pfree(p);
return result;
}
#endif
return pq_putmessage(msgtype, str, slen + 1);
@ -299,12 +303,12 @@ pq_getint(int *result, int b)
int
pq_getstr(StringInfo s)
{
int c;
int result;
#ifdef MULTIBYTE
char *p;
#endif
c = pq_getstring(s);
result = pq_getstring(s);
#ifdef MULTIBYTE
p = (char *) pg_client_to_server((unsigned char *) s->data, s->len);
@ -314,8 +318,9 @@ pq_getstr(StringInfo s)
s->len = 0;
s->data[0] = '\0';
appendBinaryStringInfo(s, p, strlen(p));
pfree(p);
}
#endif
return c;
return result;
}

76
src/backend/utils/mb/mbutils.c
View File

@ -3,7 +3,7 @@
* client encoding and server internal encoding.
* (currently mule internal code (mic) is used)
* Tatsuo Ishii
* $Id: mbutils.c,v 1.8 1999/07/17 20:18:10 momjian Exp $ */
* $Id: mbutils.c,v 1.9 1999/09/11 22:28:00 tgl Exp $ */
#include "postgres.h"
@ -93,59 +93,81 @@ pg_get_client_encoding()
}
/*
* convert client encoding to server encoding. if server_encoding ==
* client_encoding or no conversion function exists,
* returns s. So be careful.
* convert client encoding to server encoding.
*
* CASE 1: if no conversion is required, then the given pointer s is returned.
*
* CASE 2: if conversion is required, a palloc'd string is returned.
*
* Callers must check whether return value differs from passed value
* to determine whether to pfree the result or not!
*
* Note: we assume that conversion cannot cause more than a 4-to-1 growth
* in the length of the string --- is this enough?
*/
unsigned char *
pg_client_to_server(unsigned char *s, int len)
{
static unsigned char b1[MAX_PARSE_BUFFER * 4]; /* is this enough? */
static unsigned char b2[MAX_PARSE_BUFFER * 4]; /* is this enough? */
unsigned char *p = s;
unsigned char *result = s;
unsigned char *buf;
if (client_encoding == GetDatabaseEncoding())
return (p);
return result;
if (client_to_mic)
{
(*client_to_mic) (s, b1, len);
len = strlen(b1);
p = b1;
buf = (unsigned char *) palloc(len * 4 + 1);
(*client_to_mic) (result, buf, len);
result = buf;
len = strlen(result);
}
if (server_from_mic)
{
(*server_from_mic) (p, b2, len);
p = b2;
buf = (unsigned char *) palloc(len * 4 + 1);
(*server_from_mic) (result, buf, len);
if (result != s)
pfree(result); /* release first buffer */
result = buf;
}
return (p);
return result;
}
/*
* convert server encoding to client encoding. if server_encoding ==
* client_encoding or no conversion function exists,
* returns s. So be careful.
* convert server encoding to client encoding.
*
* CASE 1: if no conversion is required, then the given pointer s is returned.
*
* CASE 2: if conversion is required, a palloc'd string is returned.
*
* Callers must check whether return value differs from passed value
* to determine whether to pfree the result or not!
*
* Note: we assume that conversion cannot cause more than a 4-to-1 growth
* in the length of the string --- is this enough?
*/
unsigned char *
pg_server_to_client(unsigned char *s, int len)
{
static unsigned char b1[MAX_PARSE_BUFFER * 4]; /* is this enough? */
static unsigned char b2[MAX_PARSE_BUFFER * 4]; /* is this enough? */
unsigned char *p = s;
unsigned char *result = s;
unsigned char *buf;
if (client_encoding == GetDatabaseEncoding())
return (p);
return result;
if (server_to_mic)
{
(*server_to_mic) (s, b1, len);
len = strlen(b1);
p = b1;
buf = (unsigned char *) palloc(len * 4 + 1);
(*server_to_mic) (result, buf, len);
result = buf;
len = strlen(result);
}
if (client_from_mic)
{
(*client_from_mic) (p, b2, len);
p = b2;
buf = (unsigned char *) palloc(len * 4 + 1);
(*client_from_mic) (result, buf, len);
if (result != s)
pfree(result); /* release first buffer */
result = buf;
}
return (p);
return result;
}
/* convert a multi-byte string to a wchar */

8
src/include/config.h.in
View File

@ -123,14 +123,6 @@
*/
/* #define TCL_ARRAYS */
/*
* The following flag allows copying tables from files with number of columns
* different than the number of attributes setting missing attributes to NULL
* and ignoring extra columns. This also avoids the shift of the attributes
* of the rest of the file if one line has a wrong column count.
*/
#define COPY_PATCH
/*
* User locks are handled totally on the application side as long term
* cooperative locks which extend beyond the normal transaction boundaries.