new map format

[Xaphier]

Hello everybody!

 

I would like to program a new map format for el (Todo & Projects, Can't find something to do? Task 5). I would use a height field, multiple textures, water, bump maps, vertex buffer and the 3d & 2d objects that are currently used. The bump maps and the vertex buffer are optional, so the user could switch them off if his graphic card doesn't support them. Is that what you want?

[Wytter]

I would like to use sub-divided surfaces to create smooth terrain. That way we could implement LOD as well, depending on the distance from the camera.

 

The height field would be similiar in size to the current height-map (perhaps 4 times the size depending on the results)

The height map will still be a different array, but you could change it automagically when terrain is lowered/raised - the heightmap is used on the server to determine the characters position. The terrain map could be put in a different file (at least in the beginning).

 

We're in the process of changing the map format - so far we've planned this, however it is very persistent to change:

 

struct map_header {
-  Uint8  file_sig[4] = "elmf"
-
-  Uint32 tile_map_x_len       /* moved four bytes forward */
-  Uint32 tile_map_y_len       /* moved four bytes forward */
-  Uint32 tile_map_offset      /* moved four bytes forward */
-
-  Uint32 height_map_offset    /* moved 64 bytes forward */
+  Uint8  file_sig[4] = "elmg" /* change magic number to differentiate from
+                                  versionless predecessor */
+  Uint8  format_version[4] = { 1,1,0,0 } /* 1.1.0.0, version number independet of client */
+
+  Uint32 tile_map_x_len       
+  Uint32 tile_map_y_len
+  Uint32 tile_map_offset
  
  Uint32 obj_3d_struct_len
  Uint32 obj_3d_no
  Uint32 obj_3d_offset

  Uint32 obj_2d_struct_len
  Uint32 obj_2d_no
  Uint32 obj_2d_offset

  Uint32 lights_struct_len
  Uint32 lights_no
  Uint32 lights_offset

-  Uint8  dungeon
-  Uint8  (reserved)
-  Uint8  (reserved)
-  Uint8  (reserved)
+  Uint32 flags
  
  float  ambient_r
  float  ambient_g
  float  ambient_b
  
  Uint32 particles_struct_len
  Uint32 particles_no
  Uint32 particles_offset
  
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
-  Uint32 (reserved)
+  Uint32 height_map_offset
+  Uint32 sector_map_offset
+
+  Uint32 animation_struct_len
+  Uint32 animation_no
+  Uint32 animation_offset
+
+  Uint8  continent
+  Uint8  continent_map_len     // size on continent map (LSB stripped)
+  Uint8  continent_map_x_pos   // position on continent map (LSB stripped)
+  Uint8  continent_map_y_pos   // position on continent map (LSB stripped)
+  Uint8  (reserved)[16]
} // struct map_header

struct obj_2d_io {
-  Uint8  file_name[80]
+  Uint32 cls_id      /* object class from e2dlist.txt */
+  Uint8  (reserved)[76] /* for easier conversion */
  
  float  x_pos
  float  y_pos
  float  z_pos
  
  float  x_rot
  float  y_rot
  float  z_rot
  
  Uint8 (reserved)[24]
} // struct obj_2d_io

struct obj_3d_io {
-  Uint8  file_name[80]
+  Uint32 cls_id         /* object class from e3dlist.txt */
+  Uint32 obj_id         /* object id for .def files */
+  Uint32 flags
+  float  r,g,b
+  Uint8  (reserved)[56] /* for easier conversion */
  
  float  x_pos
  float  y_pos
  float  z_pos
  
  float  x_rot
  float  y_rot
  float  z_rot

-  Uint8  self_lit        /* included in flags     */
-  Uint8  blended         /* included in flags     */
+  Uint8  (reserved)[2]
-  float  r,g,b           /* improved alignment */
-  Uint8  (reserved)[24]
+  Uint8  (reserved)[32]
} // struct obj_3d_io {

struct light_io {
  float  pos_x
  float  pos_y
  float  pos_z
  float  r,g,b
} // struct light_io

struct particles_io {
-  Uint8  file_name[80]
+  Uint32 cls_id         /* object class from particles.txt */
+  Uint8  (reserved)[76] /* for easier conversion */
  
  float  x_pos
  float  y_pos
  float  z_pos
  
  Uint8  (reserved)[10]
} // struct particles_io

+/*!
+ * Defines an animated object
+ */
+struct animation_io {
+  Uint32 cls_id /* object class */
+  Uint32 obj_id /* object id for .def files */
+  Uint32 flags  /* includes self-lit */
+  float  r,g,b  /* if self-lit */
+  Uint8 (reserved)[56] /* make it the same size as obj_3d_io */
+ 
+  float  x_pos
+  float  y_pos
+  float  z_pos
+ 
+  float  x_rot
+  float  y_rot
+  float  z_rot
+
+  Uint8  (reserved)     /* make it the same size as obj_3d_io */
+  Uint8  blend_func
+  Uint8  (reserved)[32] /* make it the same size as obj_3d_io */
+} // struct animation_io
+
+/*!
+ * The bit structure of a height map tile,
+ *   total size is 16 bits
+ */
+struct height_map_tile_io {
+  Uint8  (reserved) :  3 /* internal server flags */
+  Uint8  walkable   :  1
+  Uint8  swimmable  :  1
+  Uint8  flyable    :  1
+  Uint16 height     : 10
+}
+
+/*!
+ * The height map structure for one level of one sector
+ */
+typedef height_map_tile_io sector_height_map_io[6*TILES_PER_SECTOR][6*TILES_PER_SECTOR]
+
+/*!
+ * The sector info structure.
+ */
+struct sector_io {
+  Uint8 num_levels
+}
+
+/*!
+ * \name map flags
+ */
+/*! \{ */
+#define DUNGEON     0x00000001
+/*! \} */
+
+/*!
+ * \name 3d object flags
+ */
+/*! \{ */
+#define HARVESTABLE 0x00000009
+#define MINABLE     0x00000001 // for future use
+#define (reserved)  0x00000002 // used in barren moon
+#define (reserved)  0x00000004 // used in barren moon
+#define PICKABLE    0x00000008 // for future use
+#define USABLE      0x00000010
+#define USABLE_WITH 0x00000020
+#define (reserved)  0x00000040 // used in barren moon
+#define ENTRABLE    0x00000080
+#define (reserved)  0x00000100 // used in barren moon
+#define STORAGE     0x00000200 // new storage interface (server-side)
+#define BLENDED     0x40000000 // enable alpha blending
+#define SELF_LIT    0x80000000 // disable lighting
+/*! \} */

[Xaphier]

LOD is not good, it is slow and the map will look bad. Perhaps we do this with displacement mapping. But it is not fast and just new graphic cards can do it.

I think we should use bump maps, they are faster and use less system and graphic memory.

For older graphic cards we can fake bump maps with a vertex shader/program and register combiners. This will work on a Geforce 1/2.

[Wytter]

Yes, I see that using bump maps is possible - however we need a fall-back method for older graphics cards than GF 1/2.

 

Were you planning to just use the verticies specified by the terrain map and then use bump mapping to make it look smoother?

[Xaphier]

Yes, I think that is it.

[Wytter]

I think that it would be acceptable to have somewhat hard edges on very old graphic cards - at least at first, we can always come up with other ways to make the terrain look somewhat smooth on older cards.

 

Before you start taking this task, be advised that the map editor code can be quite tricky to work with, and we were planning a total rewrite (where we basically take the code from ELC, remove whatever we don't need and add the map editor to it). So don't worry about rewriting the old code, it will be done - just concentrate on the terrain.

 

Which operating system and compiler are you currently using?

 

When adding the terrain, please use a #define to differ between the version with and without terrain.

 

When raising or lowering the terrain in the map editor, the height map should automatically be moved as well.

[Xaphier]

My system is Linux Gentoo on AMD64, compiler is gcc 3.4.3

[Wytter]

Good - same as mine then. Which graphics card are you using?

 

Make sure to compile with -Wall -Wdeclaration-after-statement -Werror.

Please try to stay up to date with CVS, it will make it hundred of times easier to do the final merge then.

 

If you want some doxygen documentation of the client, do a "make docs".

 

If you need help, or have questions feel free to contact me. I will be gone from the 22nd till the 27th otherwise I should be fairly easy to reach.

 

Anyway, I wish you good luck

[Xaphier]

My graphic card is Geforce 6600 GT.

I just need some infos

Which programming language can I sue? I think plain C but I am not sure.

Which library can I use? I would like to use zlib for compression.

Which OpenGL version can I use?

Which programming language can I sue for the vertex and pixel shader? I would like to use the OpenGL shading language.

[Wytter]

We use plain C - or rather a mix between C89 and C99. The main compiler is GCC on Linux and Windows, and we aim at supporting GCC 2.95 and up. This means that you can use some C99 features such as inlining, but you still have to declare your variables at the beginning of the scope. This means that:

 

int main(int argc, char ** argv)<br>
{
      if(argc<2){
             printf("You have to provide at least 1 argument\n");
             return 0;
      }

      int l;

      for(l=0;l<argc;l++){
             printf("%d: %s\n",l,argv[l]);
      }

      return 1;
}

is invalid, whereas

int main(int argc, char ** argv)
{
      int l;

      if(argc<2){
             printf("You have to provide at least 1 argument\n");
             return 0;
      }

      for(l=0;l<argc;l++){
             printf("%d: %s\n",l,argv[l]);
      }

      return 1;
}

is valid since l is declared at the beginning of the scope. It just means that vars have to be declared after a {, and not after you've started writing your code.

 

Indenting is a matter of personal preference - just make sure that you remember to indent, otherwise it can make the code almost unreadable.

 

You may use zlib for compression - afaik the libxml2 binary for Windows uses this as well + we have been considering using it for some data transfers from the server to the client. If you wish to use more libraries, please ask

 

Since we aim at supporting a range of different platforms and both older and newer graphics cards you will have to have fallback methods for older cards as well; so if an extension is not found, it should still be able to run the game, although with worse graphics.

 

You may use GLSL, ARBfp1.0 and/or ARBvp1.0. Preferably GLSL.

 

I'll add the function pointers needed for creating/deleting programs to the client later today.

[Grum]

Indenting is a matter of personal preference

Although your code will probably not be rejected, I would like to *strongly* encourage you to use one tab character for indentation.

[Wytter]

I agree

 

Just added support for using the GL_EXT_framebuffer_object, GL_ARB_fragment_program, GL_ARB_vertex_program, GL_ARB_vertex_shader, GL_ARB_fragment_shader, GL_ARB_shading_language_100, GL_ARB_shader_objects extensions:

void (APIENTRY * ELglGenRenderbuffersEXT)(GLsizei n, GLuint * renderbuffers);
void (APIENTRY * ELglDeleteRenderbuffersEXT)(GLsizei n, const GLuint * renderbuffers);
void (APIENTRY * ELglBindRenderbufferEXT)(GLenum target, GLuint renderbuffer);
void (APIENTRY * ELglRenderbufferStorageEXT)(GLenum target, GLenum internalformat, GLsizei width, GLsizei height);
void (APIENTRY * ELglGenFramebuffersEXT)(GLsizei n, GLuint * framebuffers);
void (APIENTRY * ELglDeleteFramebuffersEXT)(GLsizei n, const GLuint * framebuffers);
void (APIENTRY * ELglBindFramebufferEXT)(GLsizei n, GLuint framebuffer);
void (APIENTRY * ELglGenProgramsARB)(GLsizei n, GLuint * programs);
void (APIENTRY * ELglDeleteProgramsARB)(GLsizei n, const GLuint * programs);
void (APIENTRY * ELglBindProgramARB)(GLenum type, GLuint program);
void (APIENTRY * ELglProgramStringARB)(GLenum type, GLenum format, GLsizei length, const char * program);
void (APIENTRY * ELglDeleteObjectARB)(GLhandleARB obj);
GLhandleARB (APIENTRY * ELglCreateShaderObjectARB)(GLenum type);
void (APIENTRY * ELglShaderSourceARB)(GLhandleARB shader, GLsizei count, const char ** string, const int * length);
void (APIENTRY * ELglCompileShaderARB)(GLhandleARB shader);
GLhandleARB (APIENTRY * ELglCreateProgramObjectARB)(void);
void (APIENTRY * ELglAttachObjectARB)(GLhandleARB program, GLhandleARB shader);
void (APIENTRY * ELglLinkProgramARB)(GLhandleARB program);
void (APIENTRY * ELglUseProgramObjectARB)(GLhandleARB program);

[Lachesis]

Although your code will probably not be rejected, I would like to *strongly* encourage you to use one tab character for indentation.

188238[/snapback]

(nodds avidly) ^_^

[Brom]

Is there not a code-clean up program? Like Jacobe for java?

http://www.tiobe.com/jacobe.htm

[crusadingknight]

Is there not a code-clean up program? Like Jacobe for java?

http://www.tiobe.com/jacobe.htm

188794[/snapback]

An indentor/formatter can't ever make up entirely for good coding style.

Edited August 18, 2005 by crusadingknight

[Lachesis]

Run an indentor on CVS code and commit the result. I hope you'll find a good place to hide afterwards

[Xaphier]

My main problem at the moment is memory usage. If TILES_PER_SECTOR is 4, every tile uses at least 1152 Byte of height map data. This is 72MB for big maps (256x256 tiles).

The normal maps for bump/normal mapping are even bigger (1728 bytes per tile). We can calculate the normal map at runtime, but this is not cheap.

So, the main questions is: How much memory and disk space can is use for height maps and perhaps bump/normal maps :icon14:

 

PS: zlib will save disk space

[Entropy]

A walk tile (a step) is 50 cm. A normal tile (say, grass) has 3x3 meters. I want no more than 1 vertex per 25 cm. More than that is too much memory wasted.

So then for a normal tile we have:

number of vertices * 4 (a float is 4 bytes) * (x + y + z + u + v + texture)

that is:

144*4*6=3456

For a 64x64 map we'll have: 14,155,776

There is no rule in the unverse dictating that we have to use maps bigger than 64x64. Keep in mind that the current map format (an array for the tiles) will be kept, but we want to add a new, heightmap format as well.

[Xaphier]

OK. I thought maps could get bigger than 64x64 tiles. My fault.

If TILES_PER_SECTOR is 4, we could use 36 vertexes per tile and sixteen normals per vertex. We could scale the height map for the vertexes, so that sixteen height_map_tile_io's give one vertex. We can use the normals for bump/normal mapping to make it look smoother. This is very cheap at modern graphic cards and we can use dot3 texture blending on older graphic cards ore just disabling bump/normal maps on this cards.

At runtime, the height map will be saved as floats in memory, the normal map as colours (3 bytes per colour). This gives 144 bytes (36 vertexes per tile, every tile just use one float) per tile for the vertexes and 1728 bytes (36(vertexes per tile)*16(normals per vertex)*3(bytes per normal) per tile for the normal map. This is smaller than 8MB for 64x64 maps. So we can calculate the normal map at runtime, because we yust do it once(loading the map) and keep it in memory.

[Entropy]

One thing that I would REALLY like is to have at least two textures at the same time (multitexturing). Otherwise it will look crappy and will be very hard to make stuff such as roads.

[Xaphier]

Multitexturing is no problem

[Entropy]

It will make the map larger though, because we need to store extra info, such as the u/v of two textures and the ID of those two textures.

[Xaphier]

Why not using 2D objects :icon14:

[Entropy]

Because the 2d objects are not designed to wrap on the terrain and stuff.

[Xaphier]

OK. We could do it like this.

#define VERTEXES_PER_TILE  6*6
#define NORMALS_PER_VERTEX	4*4

struct height_map_tile_io {
   Uint8  (reserved) :  3 /* internal server flags */
   Uint8  walkable   :  1
   Uint8  swimmable  :  1
   Uint8  flyable    :  1
   Uint16 height     : 10
}

struct tile {
   height_map_tile_io height_map[VERTEXES_PER_TILE*NORMALS_PER_VERTEX];
   float u_texture_1[VERTEXES_PER_TILE];
   float v_texture_1[VERTEXES_PER_TILE];
   float u_texture_2[VERTEXES_PER_TILE];
   float v_texture_2[VERTEXES_PER_TILE];
   Uint16 water_height;
   Uint8 base_texture, texture_1, texture_2, water_texture;
}

With water_height we can make rivers, sees etc. without extra 3D-objects.