How to properly mask/unmask frames (WebSocket)?

Question

C++ / C#

We implement a WebSocket server in c++. The handshake goes smoothly. The received data is successfully decoded on the server. The problem is when sending data to the browser, an undefined error constantly crashes and the connection is closed with error 1006. Tell me what we are doing wrong. Function code:

int WEBSOCKET_set_content( const char *data, int data_length, unsigned char *dst) 
{
  unsigned char *message = ( unsigned char * )malloc( 65535 * sizeof( char ) );
  int i;
  int data_start_index;

  message[0] = 129;

  if( data_length <= 125 ) 
  {
    message[1] = ( unsigned char )data_length;
    data_start_index = 2;
  } 
  else 
    if( data_length > 125 && data_length <= 65535 ) 
    {
      message[1] = 126;
      message[2] = ( unsigned char )( ( data_length >> 8 ) & 255 );
      message[3] = ( unsigned char )( ( data_length ) & 255 );
      data_start_index = 4;
    } 
    else 
    {
      message[1] = 127;
      message[2] = ( unsigned char )( ( data_length >> 56 ) & 255 );
      message[3] = ( unsigned char )( ( data_length >> 48 ) & 255 );
      message[4] = ( unsigned char )( ( data_length >> 40 ) & 255 );
      message[5] = ( unsigned char )( ( data_length >> 32 ) & 255 );
      message[6] = ( unsigned char )( ( data_length >> 24 ) & 255 );
      message[7] = ( unsigned char )( ( data_length >> 16 ) & 255 );
      message[8] = ( unsigned char )( ( data_length >> 8 ) & 255 );
      message[9] = ( unsigned char )( ( data_length ) & 255 );
      data_start_index = 10;
    }

  for( i = 0; i < data_length; i++ ) 
  {
    message[ data_start_index + i ] = ( unsigned char )data[i];
  }

  for( i = 0; i < data_length+ data_start_index; i++ ) 
  {
    dst[i] = ( unsigned char )message[ i ];
  }

  if( message ) 
  {
    free( message );
    message = NULL;
  }

  return i;
}

int WEBSOCKET_get_content( const char *data, int data_length, unsigned char *dst) 
{
  unsigned int i, j;
  unsigned char mask[4];
  unsigned int packet_length = 0;
  unsigned int length_code = 0;
  int index_first_mask = 0;
  int index_first_data_byte = 0;

  if( ( unsigned char )data[0] != 129 ) 
  {
    dst = NULL;
    if( ( unsigned char )data[0] == 136 ) 
    {
      /* WebSocket client disconnected */
      return -2;
    }
    /* Unknown error */
    return -1;
  }

  length_code = ((unsigned char) data[1]) & 127;

  if( length_code <= 125 ) 
  {
    index_first_mask = 2;

    mask[0] = data[2];
    mask[1] = data[3];
    mask[2] = data[4];
    mask[3] = data[5];
  } 
  else 
    if( length_code == 126 ) 
    {
      index_first_mask = 4;

      mask[0] = data[4];
      mask[1] = data[5];
      mask[2] = data[6];
      mask[3] = data[7];
    } 
    else 
      if( length_code == 127 ) 
      {
        index_first_mask = 10;

        mask[0] = data[10];
        mask[1] = data[11];
        mask[2] = data[12];
        mask[3] = data[13];
      }

  index_first_data_byte = index_first_mask + 4;

  packet_length = data_length - index_first_data_byte;

  for( i = index_first_data_byte, j = 0; i < data_length; i++, j++ ) 
  {
    dst[ j ] = ( unsigned char )data[ i ] ^ mask[ j % 4];
  }

  return packet_length;
}

Answer 1

You have two problems, one big, one small.
Big - you write in condo-nasty style, called "I'm walking in C ++ like in C, I don't care." Drop it.
Small - I see clearly that your int is 32 bit, and you shift it by 50 with a jack of bits. It's on the verge of UB, to be honest.
Throw away this filthy style of writing without cycles (this is how the Chinese sculpt firmware for fake iPhones, they are paid line by line), finally learn STL and write beautifully.