UPDATING Sybase IMAGE column from ESQL/C gives error 187 Odd number of bytes - sybase-ase

We face a problem with UPDATING Sybase IMAGE column from ESQL/C written program.
The ASE server is 15.7.
The IMAGE column is around 50K of size and was INSERTed fine, can be FETCHed fine, but
on UPDATE it gives the ASE Error 187 Odd number of bytes in IMAGE data.
I diged deep into this with a debugger gdb and watching the communication
between the ESQL/C program and the ASE server. What I see there for the UPDATE
command sent over the network to the server is the following:
...
sendto(5, "\17\0\10\0\0\0\0\0\205\20\207\v\0I\0\1\vtitel_daten\375\206UPDATE titel_daten SET aktion =#sql0_aktion , fcopy =#sql1_fcopy , reserv =#sql2_reserv , vsias =#sql3_vsias , vsiera =#sql4_vsiera , vopac =#sql5_vopac , nettodaten =
0x0002000a31382e30372e32303138005bffe0001e546869732069732074686520756c74696d6174652066616b6520626f6f6b005dffb500496d656c6f64792c206c79726963732c2063686f726473203b20666f7220 ....
0002303503d1fff2000c5244412d4175666e61686d65 \312\6\0\0\0\25\177\344\4\262\6\5\0\2\1\0\1 \1 \4 \4 \4 J\0\0\0\6\0\0\0\0\0\0\0\0\0\0&\2\0\0\0\0\0\0\0\0\0\0'\2\0\0\0\0\0\0\0\0\0\0'\2\0\0\0\0\0\0\0\0\0\0'\5\0\0\0\0\0\0\0\0\0\0'\5\0\0\0\0\0\0\0\0\0\0'\5\0\327\2\0\0\1 \1 \4 \4 \4 ",
100652, 0, NULL, 0) = 100652
From the send(5) syscall I see that the image data (for the column 'nettodaten') is packed every
byte two its hex representation, which results (together with some other INTEGER columns)
in a total data of 100652 bytes, which matches the length of the IMAGE data of 50K
(doubled to 100K). I even checked the full packed contents of the IMAGE in
the sendto() buffer: all data is there correctly encoded and packed.
The ASE server responses with the Error 187.
The ##textsize is set with SET textsize 1000000, i.e. the length of 50K or even 100K
should not be any problem (and the row could be INSERTED). With the above
UPDATE the IMAGE column should not be changed, but only some of the other
INTEGER vales will get some new values.
What is wrong?
update 1
I did systematic tests. When I reduce the number of bytes to be written into the IMAGE column to 33023, the update goes fine, with 33024 and above, it fails with the error 187.
update 2
I can reproduce the error with a short ESCQL/C pgm (see below)
The table image_test contains only two columns, one integer and one image, and one single row with katkey=1 for the update.
running
LANG=C TEXTSIZE=49957 ./example4
FETCHED: syb_identity_col [1] key [1] length [30030] blobfld []
writing 65535 zeros into CS_IMAGE blobfld
now running UPDATE with ImageLengthField [49957] ImageLengthDefine [49957] ...
** SQLCODE=(-187)
** ASE Error
** Odd number of bytes in IMAGE data.
i.e. gives the error when the size of the image is 49957 bytes, but works for example for 30030 bytes:
TEXTSIZE=30030 ./example4
FETCHED: syb_identity_col [1] key [1] length [30] blobfld [Thu Jul 26 14:21:42 CEST 2018
]
writing 65535 zeros into CS_IMAGE blobfld
now running UPDATE with ImageLengthField [30030] ImageLengthDefine [30030] ...
here is the code:
/*
** based on Sybase' example4.cp
**
*/
#include <stdio.h>
#include "sybsqlex.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
EXEC SQL INCLUDE SQLCA;
void error_handler();
void warning_handler();
void notfound_handler();
static int ImageLengthField;
#define ImageLengthDefine 65535
EXEC SQL BEGIN DECLARE SECTION;
static CS_IMAGE blobfld[ImageLengthDefine];
EXEC SQL END DECLARE SECTION;
#define ImageLengthDefine ImageLengthField
static char blob[65535];
int main(int argc, char *argv[])
{
int realLength;
int fd;
char *p;
EXEC SQL BEGIN DECLARE SECTION;
long syb_identity_col = 0;
long length = 0;
CS_INT key;
char username[30];
char password[30];
char hs_image_test[25];
char anweisung[64*1024];
EXEC SQL END DECLARE SECTION;
EXEC SQL WHENEVER SQLERROR CALL error_handler();
EXEC SQL WHENEVER SQLWARNING CALL warning_handler();
EXEC SQL WHENEVER NOT FOUND CALL notfound_handler();
strcpy(username, "sisis");
strcpy(password, "sisis123");
EXEC SQL CONNECT :username IDENTIFIED BY :password;
EXEC SQL USE imxreg;
EXEC SQL SET textsize 1000000;
EXEC SQL SET textsize 1000000;
EXEC SQL BEGIN TRANSACTION;
sprintf(anweisung, "SELECT SYB_IDENTITY_COL, katkey, nettodaten, datalength(nettodaten) FROM image_test WHERE katkey = ? FOR UPDATE");
strcpy(hs_image_test, "hs_image_test");
EXEC SQL PREPARE :hs_image_test FROM :anweisung;
EXEC SQL DECLARE hc_image_test CURSOR FOR :hs_image_test;
key = 1;
EXEC SQL OPEN hc_image_test USING :key;
key = 0;
ImageLengthField = 65535;
EXEC SQL FETCH hc_image_test INTO :syb_identity_col, :key, :blobfld, :length ;
blobfld[length] = '\0';
printf("FETCHED: syb_identity_col [%d] key [%d] length [%d] blobfld [%s]\n", syb_identity_col, key, length, blobfld);
fflush(stdout);
/* from titel_daten.c line 5447:
**
** SQL STATEMENT: 51
** EXEC SQL UPDATE titel_daten
** SET aktion =:hrec_titel_daten.aktion,
** fcopy =:hrec_titel_daten.fcopy,
** reserv =:hrec_titel_daten.reserv,
** vsias =:hrec_titel_daten.vsias,
** vsiera =:hrec_titel_daten.vsiera,
** vopac =:hrec_titel_daten.vopac,
** nettodaten = :blobfld
** WHERE CURRENT OF hc_titel_daten;
*/
/* 49957 is the length of our data in the file we want to write into the
image row 'nettodaten'
*/
realLength = 49957;
fd = open("/home/sisis/guru/titel_daten.blob.update", O_RDONLY);
if (fd >= 0) {
read(fd, blob, realLength);
blobfld[realLength] = '\0';
close(fd);
} else {
printf("can't read BLOB file /home/sisis/guru/titel_daten.blob.update\n");
exit(1);
}
printf("writing %d zeros into CS_IMAGE blobfld\n", ImageLengthField);
memset(blobfld, '\0', ImageLengthField);
memcpy(blobfld, blob, realLength);
p = getenv("TEXTSIZE");
if (p!=NULL)
realLength = atoi(p);
ImageLengthField = realLength;
printf("now running UPDATE with ImageLengthField [%d] ImageLengthDefine [%d] ... \n", ImageLengthField, ImageLengthDefine);
EXEC SQL UPDATE image_test
SET
nettodaten =:blobfld
WHERE CURRENT OF hc_image_test;
EXEC SQL CLOSE hc_image_test;
EXEC SQL COMMIT TRANSACTION;
EXEC SQL DISCONNECT DEFAULT;
return(STDEXIT);
}
/*
** void error_handler()
**
** Displays error codes and numbers from the SQLCA and exits with
** an ERREXIT status.
*/
void
error_handler(void)
{
fprintf(stderr, "\n** SQLCODE=(%ld)", sqlca.sqlcode);
if (sqlca.sqlerrm.sqlerrml)
{
fprintf(stderr, "\n** ASE Error ");
fprintf(stderr, "\n** %s", sqlca.sqlerrm.sqlerrmc);
}
fprintf(stderr, "\n\n");
exit(ERREXIT);
}
/*
** void warning_handler()
**
** Displays warning messages.
*/
void
warning_handler(void)
{
if (sqlca.sqlwarn[1] == 'W')
{
fprintf(stderr,
"\n** Data truncated.\n");
}
if (sqlca.sqlwarn[3] == 'W')
{
fprintf(stderr,
"\n** Insufficient host variables to store results.\n");
}
return;
}
/*
** void notfound_handler()
**
** Displays notfound messages.
*/
void
notfound_handler(void)
{
if (sqlca.sqlcode == 100)
{
fprintf(stderr, "No rows found\n");
}
return;
}

Related

Can Perl manage millions of files?

I like to make my own scripts to plot some data in bash.
Anyway for now I have thousands of files that I want to "rename", "edit"
and much more.
When I try to manage those files, the shell says something like
$Arguments list too long
I'm wondering if Perl can manage such amount of files?
There is an upper limit to the length of the arguments plus environment that you are allowed on Unix. For many modern versions of Unix, that limit is around 256 KiB — for other versions, it may be less.
This is a not a limit of the shell, per se, nor of Perl nor any other program, but rather a limit imposed by the Unix kernel.
If handling files one at a time, Perl can handle many millions of files in total. The difficulty is conveying the list of files to Perl. You can write the names to a file and tell Perl which file to read. You can use xargs. You have to worry about what marks the end of a file name. The safe answer is a null byte; it is the only character that cannot appear in a pathname in Unix. You'd find it easiest to use POSIX function getdelim() to read such lines. Using a newline is conventional but not 100%; a file name can contain a newline, leading to confusion.
You could also have Perl generate the lists of files itself, by reading directories (piecemeal, but doesn't drag millions of names into memory all at once) or by using functions such as glob.
See also SO 18559403: How big an argument list is allowed.
This code can help you determine the limit on your argument list; it is a refinement of an answer I gave to the cross-referenced question. It tells me 256 KiB on Mac OS X 10.8.5. On an archaic Linux 2.6 kernel, I got a limit of 128 KiB.
/* SO 18559403: How big an argument list is allowed */
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/wait.h>
#include <unistd.h>
extern char **environ; /* Sometimes in <unistd.h> */
enum { BYTES_PER_KIBIBYTE = 1024 };
enum { BYTES_PER_MEBIBYTE = BYTES_PER_KIBIBYTE * BYTES_PER_KIBIBYTE };
enum { E_GOT_E2BIG = 37 };
enum { E_NOT_E2BIG = 219 };
enum { R_TOO_LARGE = +1, R_TOO_SMALL = -1 };
static char *print_kib(int size, char *buffer, size_t buflen)
{
snprintf(buffer, buflen, "%d (%d KiB)", size, size / BYTES_PER_KIBIBYTE);
return buffer;
}
static int test_arg_size(int size)
{
char buffer[32];
int result = R_TOO_SMALL;
assert(size % 8 == 0);
fflush(0);
pid_t pid = fork();
if (pid < 0)
{
fprintf(stderr, "Failed to fork at size %s\n",
print_kib(size, buffer, sizeof(buffer)));
exit(1);
}
else if (pid == 0)
{
int self = getpid();
printf("Child: %d\n", self);
char *args[10] = { "ls" };
size_t bytes_per_arg = size / 8;
for (int j = 1; j < 9; j++)
{
args[j] = malloc(bytes_per_arg);
if (args[j] == 0)
{
fprintf(stderr, "Failed to allocate argument space at size %s\n",
print_kib(size, buffer, sizeof(buffer)));
exit(E_NOT_E2BIG);
}
memset(args[j], j + '0', bytes_per_arg - 1);
args[j][bytes_per_arg - 1] = '\0';
}
/* Close standard I/O channels so executed command doesn't spew forth */
int dev_null = open("/dev/null", O_RDWR);
if (dev_null < 0)
{
fprintf(stderr, "Failed to open /dev/null for reading and writing\n");
exit(E_NOT_E2BIG);
}
int dev_stderr = dup(2);
if (dev_stderr < 0)
{
fprintf(stderr, "Failed to dup() standard error\n");
exit(E_NOT_E2BIG);
}
close(0);
dup(dev_null);
close(1);
dup(dev_null);
close(2);
dup(dev_null);
close(dev_null);
/* Execute ls on big file names -- error is ENAMETOOLONG */
execvp(args[0], args);
/* Reinstate standard error so we can report failure */
dup2(dev_stderr, 2);
int errnum = errno;
if (errnum == E2BIG)
{
fprintf(stderr, "%d: got E2BIG (%d: %s) at size %s\n",
self, errnum, strerror(errnum),
print_kib(size, buffer, sizeof(buffer)));
exit(E_GOT_E2BIG);
}
fprintf(stderr, "%d: got errno %d (%s) at size %s\n",
self, errnum, strerror(errnum),
print_kib(size, buffer, sizeof(buffer)));
exit(E_NOT_E2BIG);
}
else
{
int self = getpid();
int corpse;
int status;
while ((corpse = waitpid(pid, &status, 0)) != -1)
{
if (!WIFEXITED(status))
printf("%d: child %d died with exit status 0x%.4X", self, corpse, status);
else
{
int statval = WEXITSTATUS(status);
printf("%d: child %d died with exit status %d: ", self, corpse, statval);
switch (statval)
{
case E_GOT_E2BIG:
printf("success: got E2BIG");
result = R_TOO_LARGE;
break;
case E_NOT_E2BIG:
printf("failed: indeterminate error in child");
break;
case 1:
printf("command exited with status 1 - it worked");
break;
default:
printf("unknown: unexpected exit status %d", statval);
break;
}
}
printf(" at size %s\n", print_kib(size, buffer, sizeof(buffer)));
fflush(stdout);
}
}
return result;
}
static int env_size(void)
{
int size = 0;
for (char **ep = environ; *ep != 0; ep++)
size += strlen(*ep) + 1;
return size;
}
int main(void)
{
int env = env_size();
int lo = 0;
int hi = BYTES_PER_MEBIBYTE;
/* Binary search -- the kilobyte slop means termination does not have to be accurate */
while (lo + 1 * BYTES_PER_KIBIBYTE < hi)
{
int mid = (lo + hi) / 2;
if (test_arg_size(mid) == R_TOO_LARGE)
hi = mid;
else
lo = mid;
}
char buffer1[32];
char buffer2[32];
printf("Environment size = %d\n", env);
printf("Best guess: maximum argument size in range %s to %s\n",
print_kib(lo + env, buffer1, sizeof(buffer1)),
print_kib(hi + env, buffer2, sizeof(buffer2)));
return 0;
}
Avoid placing the file names on the command line if you are going to have lots of filenames and hence hit the shells limits.
In addition I would avoid xargs as this will slow things down due to firing up Perl multiple times.
The best solution is to use the find library i.e. http://perldoc.perl.org/File/Find.html
One of the easiest thing that you could do is to store the files in file and process that file in your script.
Like:
$ find (your find command) > /tmp/files
$ your_prg.pl /tmp/files
#!perl
my $filelist = shift #ARGV;
open(my $fh,'<',$filelist) or die $!;
while (my $filename = <$fh>){
chomp $filename;
### do whatever want to do with the file
}

Why are odd-sized fread requests split in two?

I noticed that on Windows every time I issue an unbuffered fread() request with an odd length, it's split into 2 requests (as observed through procmon):
a) fread for my requested length-1
b) 2-byte fread for the last byte
This has an obvious performance overhead like 2 kernel requests instead of one etc.
Sample code ran on Windows 10:
#include <iostream>
#include <stdio.h>
#include <stdlib.h>
int main(int argc, char* argv[]) {
FILE* pFile;
char* buffer;
pFile = fopen(argv[0], "rb");
setbuf(pFile, nullptr);
size_t len = 3;
buffer = (char*)malloc(sizeof(char)*len);
if (len != fread(buffer, 1, len, pFile)) { fputs("Reading error", stderr); exit(3); }
free(buffer);
fclose(pFile);
return 0;
}
This results in the following procmon reported calls:
ReadFile c:\work\cpptry\Debug\cpptry.exe SUCCESS Offset: 0, Length: 2, Priority: Normal
ReadFile c:\work\cpptry\Debug\cpptry.exe SUCCESS Offset: 2, Length: 2
It seems as if Windows is incapable of issuing odd-sized requests to the file system.
What's up with that?
This is implementation artifact.
MS CRT keeps all FILEs buffered even if you tell it to don't do this. Instead file buffer is set to internal buffer with space for two bytes. This allows to keep one code path instead of two and simplifies implementation of fast path in fgetc and fputc.
#define fgetc(_stream) (--(_stream)->_cnt >= 0 ? 0xff & *(_stream)->_ptr++ : _filbuf(_stream))
Some of you are probably bothered by size of the buffer (2 bytes when quasi unbuffered), but in _fread_nolock_s function we can find optimization
witch tries to read multiplies of buffer size directly to the destination bypassing file buffer.
See fread.c in CRT sources:
/* calc chars to read -- (count/streambufsize) * streambufsize */
nbytes = (unsigned)(count - count % streambufsize);
...
nread = _read_nolock(_fileno(stream), data, nbytes);
Because the file buffer's size is equal 2, even number of bytes is read directly to the destination and eventual one byte goes through the file buffer. Sometimes there could be some bytes in the buffer that need to be transfered to destination before optimized read can take place.
Bonus: buffer size is always forced to multiple of 2.
See setvbuf.c:
/*
* force size to be even by masking down to the nearest multiple
* of 2
*/
size &= (size_t)~1;
...
/*
* CASE 1: No Buffering.
*/
if (type & _IONBF) {
stream->_flag |= _IONBF;
buffer = (char *)&(stream->_charbuf);
size = 2;
}
Code snippets above are from VC 2013 CRT.
For comparison snippets from Universal CRT 10.0.17134
read.cpp
unsigned const bytes_to_read = stream_buffer_size != 0
? static_cast<unsigned>(maximum_bytes_to_read - maximum_bytes_to_read % stream_buffer_size)
: maximum_bytes_to_read;
...
int const bytes_read = _read_nolock(_fileno(stream.public_stream()), data, bytes_to_read);
setvbuf.cpp
// Force the buffer size to be even by masking the low order bit:
size_t const usable_buffer_size = buffer_size_in_bytes & ~static_cast<size_t>(1);
...
// Case 1: No buffering:
if (type & _IONBF)
{
return set_buffer(stream, reinterpret_cast<char*>(&stream->_charbuf), 2, _IOBUFFER_NONE);
}
And snippets from VC 6.0 (1998)
read.c
/* calc chars to read -- (count/bufsize) * bufsize */
nbytes = ( bufsize ? (count - count % bufsize) : count );
nread = _read(_fileno(stream), data, nbytes);
setvbuf.c
/*
* force size to be even by masking down to the nearest multiple
* of 2
*/
size &= (size_t)~1;
...
/*
* CASE 1: No Buffering.
*/
if (type & _IONBF) {
stream->_flag |= _IONBF;
buffer = (char *)&(stream->_charbuf);
size = 2;
}

sending characters from parent to child process and returning char count to parent in C

So for an assignment I have for my Computer Systems class, I need to type characters in the command line when the program runs.
These characters (such as abcd ef) would be stored in argv[].
The parent sends these characters one at a time through a pipe to the child process which then counts the characters and ignores spaces. After all the characters are sent, the child then returns the number of characters that it counted for the parent to report.
When I try to run the program as it is right now, it tells me the value of readIn is 4, the child processed 0 characters and charCounter is 2.
I feel like I'm so close but I'm missing something important :/ The char array for a and in the parent process was an attempt to hardcode the stuff in to see if it worked but I am still unsuccessful. Any help would be greatly appreciated, thank you!
// Characters from command line arguments are sent to child process
// from parent process one at a time through pipe.
//
// Child process counts number of characters sent through pipe.
//
// Child process returns number of characters counted to parent process.
//
// Parent process prints number of characters counted by child process.
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h> // for fork()
#include <sys/types.h> // for pid_t
#include <sys/wait.h> // for waitpid()
int main(int argc, char **argv)
{
int fd[2];
pid_t pid;
int status;
int charCounter = 0;
int nChar = 0;
char readbuffer[80];
char readIn = 'a';
//char a[] = {'a', 'b', 'c', 'd'};
pipe(fd);
pid = fork();
if (pid < 0) {
printf("fork error %d\n", pid);
return -1;
}
else if (pid == 0) {
// code that runs in the child process
close(fd[1]);
while(readIn != 0)
{
readIn = read(fd[0], readbuffer, sizeof(readbuffer));
printf("The value of readIn is %d\n", readIn);
if(readIn != ' ')
{
charCounter++;
}
}
close(fd[0]);
//open(fd[1]);
//write(fd[1], charCounter, sizeof(charCounter));
printf("The value of charCounter is %d\n", charCounter);
return charCounter;
}
else
{
// code that runs in the parent process
close(fd[0]);
write(fd[1], &argv, sizeof(argv));
//write(fd[1], &a, sizeof(a));
close(fd[1]);
//open(fd[0]);
//nChar = read(fd[0], readbuffer, sizeof(readbuffer));
nChar = charCounter;
printf("CS201 - Assignment 3 - Andy Grill\n");
printf("The child processed %d characters\n\n", nChar);
if (waitpid(pid, &status, 0) > 0)
{
if (WIFEXITED(status))
{
}
else if (WIFSIGNALED(status))
{
}
}
return 0;
}
}
You're misusing pipes.
A pipe is a unidirectional communication channel. Either you use it to send data from a parent process to a child process, or to send data from a child process to the parent. You can't do both - even if you kept the pipe's read and write channels open on both processes, each process would never know when it was its turn to read from the pipe (e.g. you could end up reading something in the child that was supposed to be read by the parent).
The code to send the characters from parent to child seems mostly correct (more details below), but you need to redesign child to parent communication. Now, you have two options to send the results from child to parent:
Use another pipe. You set up an additional pipe before forking for child-to-parent communication. This complicates the design and the code, because now you have 4 file descriptors to manage from 2 different pipes, and you need to be careful where you close each file descriptor to make sure processes don't hang. It is also probably a bit overkill because the child is only sending a number to the parent.
Return the result from the child as the exit value. This is what you're doing right now, and it's a good choice. However, you fail to retrieve that information in the parent: the child's termination status tells you the number of characters processed, you can fetch this value with waitpid(2), which you already do, but then you never look at status (which contains the results you're looking for).
Remember that a child process has its own address space. It makes no sense to try to read charCounter in the parent because the parent never modified it. The child process gets its own copy of charCounter, so any modifications are seen by the child only. Your code seems to assume otherwise.
To make this more obvious, I would suggest moving the declarations of variables to the corresponding process code. Only fd and pid need to be copied in both processes, the other variables are specific to the task of each process. So you can move the declarations of status and nChar to the parent process specific code, and you can move charCounter, readbuffer and readIn to the child. This will make it very obvious that the variables are completely independent on each process.
Now, some more specific remarks:
pipe(2) can return an error. You ignore the return value, and you shouldn't. At the very least, you should print an error message and terminate if pipe(2) failed for some reason. I also noticed you report errors in fork(2) with printf("fork error %d\n", pid);. This is not the correct way to do it: fork(2) and other syscalls (and library calls) always return -1 on error and set the errno global variable to indicate the cause. So that printf() will always print fork error -1 no matter what the error cause was. It's not helpful. Also, it prints the error message to stdout, and for a number of reasons, error messages should be printed to stderr instead. So I suggest using perror(3) instead, or manually print the error to stderr with fprintf(3). perror(3) has the added benefit of appending the error message description to the text you feed it, so it's usually a good choice.
Example:
if (pipe(fd) < 0) {
perror("pipe(2) error");
exit(EXIT_FAILURE);
}
Other functions that you use throughout the code may also fail, and again, you are ignoring the (possible) error returns. close(2) can fail, as well as read(2). Handle the errors, they are there for a reason.
The way you use readIn is wrong. readIn is the result of read(2), which returns the number of characters read (and it should be an int). The code uses readIn as if it were the next character read. The characters read are stored in readbuffer, and readIn will tell you how many characters are on that buffer. So you use readIn to loop through the buffer contents and count the characters. Something like this:
readIn = read(fd[0], readbuffer, sizeof(readbuffer));
while (readIn > 0) {
int i;
for (i = 0; i < readIn; i++) {
if (readbuffer[i] != ' ') {
charCounter++;
}
}
readIn = read(fd[0], readbuffer, sizeof(readbuffer));
}
Now, about the parent process:
You are not writing the characters into the pipe. This is meaningless:
write(fd[1], &argv, sizeof(argv));
&argv is of type char ***, and sizeof(argv) is the same as sizeof(char **), because argv is a char **. Array dimensions are not kept when passed into a function.
You need to manually loop through argv and write each entry into the pipe, like so:
int i;
for (i = 1; i < argv; i++) {
size_t to_write = strlen(argv[i]);
ssize_t written = write(fd[1], argv[i], to_write);
if (written != to_write) {
if (written < 0)
perror("write(2) error");
else
fprintf(stderr, "Short write detected on argv[%d]: %zd/zd\n", i, written, to_write);
}
}
Note that argv[0] is the name of the program, that's why i starts at 1. If you want to count argv[0] too, just change it to start at 0.
Finally, as I said before, you need to use the termination status fetched by waitpid(2) to get the actual count returned by the child. So you can only print the result after waitpid(2) returned and after making sure the child terminated gracefully. Also, to fetch the actual exit code you need to use the WEXITSTATUS macro (which is only safe to use if WIFEXITED returns true).
So here's the full program with all of these issues addressed:
// Characters from command line arguments are sent to child process
// from parent process one at a time through pipe.
//
// Child process counts number of characters sent through pipe.
//
// Child process returns number of characters counted to parent process.
//
// Parent process prints number of characters counted by child process.
#include <stdlib.h>
#include <stdio.h>
#include <string.h> // for strlen()
#include <unistd.h> // for fork()
#include <sys/types.h> // for pid_t
#include <sys/wait.h> // for waitpid()
int main(int argc, char **argv)
{
int fd[2];
pid_t pid;
if (pipe(fd) < 0) {
perror("pipe(2) error");
exit(EXIT_FAILURE);
}
pid = fork();
if (pid < 0) {
perror("fork(2) error");
exit(EXIT_FAILURE);
}
if (pid == 0) {
int readIn;
int charCounter = 0;
char readbuffer[80];
if (close(fd[1]) < 0) {
perror("close(2) failed on pipe's write channel");
/* We use abort() here so that the child terminates with SIGABRT
* and the parent knows that the exit code is not meaningful
*/
abort();
}
readIn = read(fd[0], readbuffer, sizeof(readbuffer));
while (readIn > 0) {
int i;
for (i = 0; i < readIn; i++) {
if (readbuffer[i] != ' ') {
charCounter++;
}
}
readIn = read(fd[0], readbuffer, sizeof(readbuffer));
}
if (readIn < 0) {
perror("read(2) error");
}
printf("The value of charCounter is %d\n", charCounter);
return charCounter;
} else {
int status;
if (close(fd[0]) < 0) {
perror("close(2) failed on pipe's read channel");
exit(EXIT_FAILURE);
}
int i;
for (i = 1; i < argc; i++) {
size_t to_write = strlen(argv[i]);
ssize_t written = write(fd[1], argv[i], to_write);
if (written != to_write) {
if (written < 0) {
perror("write(2) error");
} else {
fprintf(stderr, "Short write detected on argv[%d]: %zd/%zd\n", i, written, to_write);
}
}
}
if (close(fd[1]) < 0) {
perror("close(2) failed on pipe's write channel on parent");
exit(EXIT_FAILURE);
}
if (waitpid(pid, &status, 0) < 0) {
perror("waitpid(2) error");
exit(EXIT_FAILURE);
}
if (WIFEXITED(status)) {
printf("CS201 - Assignment 3 - Andy Grill\n");
printf("The child processed %d characters\n\n", WEXITSTATUS(status));
} else if (WIFSIGNALED(status)) {
fprintf(stderr, "Child terminated abnormally with signal %d\n", WTERMSIG(status));
} else {
fprintf(stderr, "Unknown child termination status\n");
}
return 0;
}
}
Some final notes:
The shell splits arguments by spaces, so if you start the program as ./a.out this is a test, the code will not see a single space. This is irrelevant, because spaces are supposed to be ignored anyway, but if you want to test that the code really ignores spaces, you need to quote the parameters so that the shell does not process them, as in ./a.out "this is a test" "hello world" "lalala".
Only the rightmost (least significant) 8 bits of a program's exit code are used, so WEXITSTATUS will never return more than 255. If the child reads more than 255 characters, the value will wrap around, so you effectively have a character counter modulo 256. If this is a problem, then you need to go with the other approach and set up a 2nd pipe for child-to-parent communication and write the result there (and have the parent read it). You can confirm this on man 2 waitpid:
WEXITSTATUS(status)
returns the exit status of the child. This consists of the least
significant 8 bits of the status argument that the child
specified in a call to exit(3) or _exit(2) or as the argument for a return
statement in main(). This macro should be employed only if
WIFEXITED returned true.

Malloc errors when using POSIX threads on OSX

I am currently completing final year assignments for my BSc in Computer Science one of which is on concurrency using POSIX threads.
The assignment is a very simple http server that handles http requests in separate threads and returns simple html files. I have been experiencing some signal abort errors when freeing and nullifying the char buffers at the end of the thread functions. I am working in X-Code and when I enable Guard Malloc in the scheme it works just fine. Also if I remove the calls to free() and the nullification the program runs fine too. The latter I am assuming is consuming the memory used in the previous threads and not freeing it up for other processes and therefore is not a good solution, correct? I do not understand Guard Malloc but as I will hand in a raw .cpp file for compilation on the Lecturer's Linux machine the support will not be compiled.
Below is the complete code. Any comments, style critique or pearls of wisdom of any kind you are willing to throw my way are deeply appreciated.
The input that is crashing this is the .html homepage from this random website - http://www.budgie-info.com/
Many thanks in advance.
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <pthread.h>
// Define the port number to identify this process
#define MYPORT 3490
typedef struct socket_fd {
unsigned fd;
} sock_fd;
void *request(void *fd);
int main() {
int s;
//unsigned fd;
struct sockaddr_in my_addr;
pthread_t t;
int retval;
sock_fd *s_fd;
// Construct address information
my_addr.sin_family = AF_INET;
my_addr.sin_port = htons(MYPORT);
my_addr.sin_addr.s_addr = INADDR_ANY;
memset(my_addr.sin_zero, '\0', sizeof(my_addr.sin_zero) );
// Create a socket and bind it the port MYPORT
s=socket(PF_INET,SOCK_STREAM, 0);
int yes = 1;
setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int));
retval = bind(s, (struct sockaddr *)&my_addr, sizeof(my_addr));
if( retval != 0) {
fputs ("bind() failed.",stderr);
exit (7);
}
printf("%d\n", my_addr.sin_port);
retval = listen(s,10);
if( retval != 0 ) { // Allow up to 10 incoming connections
fputs ("listen() failed.",stderr);
exit (9);
}
while(1) {
s_fd = (sock_fd*)malloc(sizeof(sock_fd));
s_fd->fd=accept(s,NULL,NULL); // wait for a request
retval = pthread_create( &t, NULL, request, s_fd);
free(s_fd);
s_fd = NULL;
if(retval != 0) {
printf("pthread_create() failed. error %d\n", retval);
}
}
}
void *request(void *s_fd) {
int retval;
char data[65536];
char header[]="HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n";
char header_gif[]="HTTP/1.1 200 OK\r\nContent-Type: image/gif\r\n\r\n";
char header_jpeg[]="HTTP/1.1 200 OK\r\nContent-Type: image/jpeg\r\n\r\n";
char header_png[]="HTTP/1.1 200 OK\r\nContent-Type: image/png\r\n\r\n";
int header_len;
char *send_buffer;
char *temp_buffer;
char filename[256];
char filename_parsed[256];
FILE *f;
sock_fd *fd_t = (sock_fd*)s_fd;
unsigned fd = fd_t->fd;
int cnt = 0;
do {
retval=(int)recv(fd,data,65536,0); // recieve the request using fd
++cnt;
} while(retval == 65536);
if(retval < 0) {
fputs ("recv() failed.\n",stderr);
return NULL;
}
data[retval]='\0'; // NUL terminate it
retval = sscanf((char *)data,"GET /%s ",filename); // get the name of the file
if(retval != 1) {
fputs ("sscanf() failed.\n",stderr);
return NULL;
}
if(strlen(filename) > 256) {
fputs ("Filename overflow.\n",stderr);
return NULL;
}
// parse uml spaces out of filenames
int j = 0;
for(int i = 0; filename[i]!='\0'; ++i) {
if(filename[i] == '%') {
filename_parsed[j] = ' ';
i+=2;
} else {
filename_parsed[j] = filename[i];
}
++j;
}
filename_parsed[j] = '\0';
//print received header
printf("Receiving:\n %s\n", data);
//print requested filename
printf("\n\n-------------filename = %s|\n\n", filename_parsed);
if( (f=fopen(filename_parsed,"rb")) == NULL ) { // open the file (might be binary)
fputs ("fopen() failed.\n",stderr);
return NULL;
}
// obtain file size:
size_t lSize;
fseek (f , 0 , SEEK_END);
lSize = ftell (f);
rewind (f);
// pre calculate length of filename
int len = (int)strlen(filename_parsed);
// identify appropriate header, alocate required memory and copy the header into the buffer
if(0 == strcmp(filename_parsed + len - 4, ".gif")) {
header_len = (int)strlen(header_gif);
printf("\n\n\n\n\nG I F\n\n\n\n");
send_buffer = (char*) malloc ( sizeof(char)*((lSize+(size_t)header_len)+14) ); // WARNING: hardcoded margin for header size differences
memcpy(send_buffer, header_gif, header_len);
} else if(0 == strcmp(filename_parsed + len - 5, ".jpeg")) {
printf("\n\n\n\n\nJ P E G\n\n\n\n");
header_len = (int)strlen(header_jpeg);
send_buffer = (char*) malloc ( sizeof(char)*((lSize+(size_t)header_len)+14) ); // WARNING: hardcoded margin for header size differences
memcpy(send_buffer, header_jpeg, header_len);
} else if(0 == strcmp(filename_parsed + len - 4, ".png")) {
header_len = (int)strlen(header_png);
printf("\n\n\n\n\nP N G \n\n\n\n");
send_buffer = (char*) malloc ( sizeof(char)*((lSize+(size_t)header_len)+14) ); // WARNING: hardcoded margin for header size differences
memcpy(send_buffer, header_png, header_len);
} else {
header_len = (int)strlen(header);
send_buffer = (char*) malloc ( sizeof(char)*((lSize+(size_t)header_len)+14) ); // WARNING: hardcoded margin for header size differences
memcpy(send_buffer, header, header_len);
}
// allocate memory to contain the whole file:
temp_buffer = (char*) malloc (sizeof(char)*(lSize+10)); // WARNING: hardcoded margin for header size differences
if (temp_buffer == NULL) {
fputs ("malloc() failed.\n",stderr);
return NULL;
}
// copy the file into the buffer:
retval = (int)fread (temp_buffer,1,lSize,f);
if (retval != lSize) {
fputs ("fread() failed.\n",stderr);
return NULL;
}
memcpy(send_buffer + header_len, temp_buffer, retval);
//print packet being sent
printf("Sending:\n%s\n", send_buffer);
memcpy(send_buffer + retval, "\r\n", 4);
// send packet
retval = (int)send(fd,send_buffer,retval,0);
if(retval < 0) {
fputs ("send() failed.\n",stderr);
return NULL;
}
free(temp_buffer); // The section of memory management causes SIGABRT errors everytime i run, commenting them out made it run smoothly.
temp_buffer = NULL; // Is the closing of this function tidying up for me or is it leaving a memory leak trail?
free(send_buffer); // Is it the multi threading that is making memory management buggy?
send_buffer = NULL;
fclose(f);
f = NULL;
close(fd); // close the socket
return NULL;
}
There are a bunch of problems with this code. It's hard to say why it's crashing on these files considering the number of potential problems (you did not provide a sample input that crashes your program)
For this problem to be usable you need to set SO_REUSEADDR on the socket, otherwise if it is crashes (hint, hint), you will not be able to restart it right away and receive connections
you do not check the return value of bind(), accept(), pthread_create()
you need to call listen() only once, not in the loop
it's safer to cast fd to long before to cast it to void * since that's what you do in listen. A cleaner way would involve a struct on the heap with an fd in it
Do not define the size of arrays that you set at compile time, let the compiler do it. It's safer: char header[]="HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n";. Then you do not need to call strlen on it. You can just use sizeof(header) - 1.
You do not check the return value of recv(). You must make sure it did not fail (esp since you use it as an index in an array)
You do not ensure that filename is not overflowed. You read up to 65K, yet you do not make sure that filename is big enough (i.e it's trivial to crash your program and it's a security vulnerability)
You do not read the return value of sscanf. There is no guarantee that filename has data
You are not nul terminating filename_parsed (I believe it's the main reason your program is crashing)
You might to call strlen(filename_parsed) only once and instead of calling it a zillion times
Instead of your confusing character checks at the end, why not use strcmp (once filename_parsed is properly terminate of course): 0 == strcmp(filename_parsed + len - 5, ".jpeg") etc.
Check the return value of send
You can check one thing: the content that's read from fread() will is not necessarifly a string - might not be NULL terminated at all. So using strcat on this is not the right way. Consider changing it to memcpy() or something.

mq_receive: message too long

I am implementing a communication between 2 processes using a queue. The problem is that when I call the function mq_receive, I get this error: Message too long.
I have done the following:
struct mq_attr attr;
long size = attr.mq_msgsize;
.... // initializing the queue "/gateway"
int rc = mq_receive(gateway, buffer, size, &prio);
If I print the size value, I get size=1, while when I print the same size but from another program (got by the same mechanism), I get something not long integer ( -1217186280 )...
How can I solve this error?....so while size = 1, I believe it's right to say "message too long" but why is 1?
P.S. I have also tried to put :
int rc = mq_receive(gateway, buffer, sizeof(buffer), &prio);
but with no result.
It seems like you need to read the docs more carefully. When you call mq_receive you should pass size of the destination buffer. This size must be greater than the mq_msgsize attribute of the queue. In addition, it seems like you have an error in queue attributes initialisation that makes proper mq_receive call impossible. Here is standard message queue session:
Fill mq_attr struct (doc):
struct mq_attr attr;
attr.mq_flags = 0;
attr.mq_maxmsg = 10;
attr.mq_msgsize = 33;
attr.mq_curmsgs = 0;
Create queue with mq_open in master process (doc):
mqd_t queue = mq_open(qname, O_CREAT|O_RDWR, 0644, &attr);
In writer process open queue for writing:
mqd_t queue = mq_open(qname, O_WRONLY);
And send some text. Length of the text must be lesser than mq_msgsize attribute of the queue (doc):
mq_send(queue, "some message", strlen("some message")+1, 1);
In reader process open queue for reading:
mqd_t queue = mq_open(qname, O_RDONLY);
And then allocate buffer and receive message. Size of buffer *must be greater than the mq_msgsize attribute of the queue. Here we create 50-byte buffer while mq_msgsize == 33 (doc):
char rcvmsg[50];
int iret = mq_receive(queue, rcvmsg, 50, NULL);
Also remember that you should use %ld for print long instead of %d.
When debugging realtime POSIX queues, you should start with a sample program which works and go forward from there. Once you have the sample program running, it should be a simple matter of ensuring that your own code follows all the steps.
The following program has been tested successfully under Ubuntu 11.04:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <mqueue.h>
#define MQNAME "/pax"
#define MQMESG "Hello there!"
static mqd_t serverUp (void) {
int rc;
mqd_t svrHndl;
struct mq_attr mqAttr;
printf ("Bringing up server.\n");
rc = mq_unlink (MQNAME);
if (rc < 0) {
printf (" Warning %d (%s) on server mq_unlink.\n",
errno, strerror (errno));
}
mqAttr.mq_maxmsg = 10;
mqAttr.mq_msgsize = 1024;
svrHndl = mq_open (MQNAME, O_RDWR|O_CREAT, S_IWUSR|S_IRUSR, &mqAttr);
if (svrHndl < 0) {
printf (" Error %d (%s) on server mq_open.\n",
errno, strerror (errno));
exit (1);
}
printf (" Server opened mqd_t of %d.\n", svrHndl);
return svrHndl;
}
static void serverReceive (mqd_t svrHndl) {
int rc;
char buffer[2048];
printf ("Server receiving on mqd_t %d.\n", svrHndl);
rc = mq_receive (svrHndl, buffer, sizeof (buffer), NULL);
if (rc < 0) {
printf (" Error %d (%s) on server mq_receive.\n",
errno, strerror (errno));
exit (1);
}
printf (" Received [%s].\n", buffer);
}
static void serverDown (mqd_t svrHndl) {
printf ("Bringing down server with mqd_t %d.\n", svrHndl);
mq_close (svrHndl);
}
static void clientSend (void) {
mqd_t cliHndl;
int rc;
printf ("Client sending.\n");
cliHndl = mq_open (MQNAME, O_RDWR);
if (cliHndl < 0) {
printf (" Error %d (%s) on client mq_open.\n",
errno, strerror (errno));
exit (1);
}
printf (" Client opened mqd_t of %d.\n", cliHndl);
rc = mq_send (cliHndl, MQMESG, sizeof (MQMESG), 1);
if (rc < 0) {
printf (" Error %d (%s) on client mq_send.\n",
errno, strerror (errno));
exit (1);
}
mq_close (cliHndl);
}
int main (void) {
mqd_t svrHndl;
svrHndl = serverUp ();
clientSend ();
serverReceive (svrHndl);
serverDown (svrHndl);
return 0;
}
The output on my system is:
Bringing up server.
Server opened mqd_t of 3.
Client sending.
Client opened mqd_t of 4.
Server receiving on mqd_t 3.
Received [Hello there!].
Bringing down server with mqd_t 3.
Don't forget to unlink the message queue before running your program again. If you dont unlink it, it will still use the old message queue settings. This happens when you end your program with Ctrl+C. I think it is a good idea to put the following code at the beginning of the program:
if(mq_unlink(QUEUENAME) == 0)
fprintf(stdout, "Message queue %s removed from system.\n", QUEUENAME);
An alternative form (C++ style) that checks for real errors (like permissions) and ignores the cases where the queue already exists or not:
int rc = mq_unlink(name.c_str());
if (rc != 0 && errno != ENOENT)
THROW_ERRNO_EXCEPTION();
// ENOENT is the status code if the queue doesn't exist, which is not an error
// if you are immediately going to create it.
Just a correction above...
"This size must be equal or greater than the mq_msgsize attribute of the queue"
Equal size is probably required if you are passing a struct instead of a buffer:
see: send struct in mq_send

Resources