OpenMotionLab/MotionGPT
118
1"""PBR renderer for Python.2 3Author: Matthew Matl4"""5import sys6 7import numpy as np8import PIL9 10from .constants import (RenderFlags, TextAlign, GLTF, BufFlags, TexFlags,11 ProgramFlags, DEFAULT_Z_FAR, DEFAULT_Z_NEAR,12 SHADOW_TEX_SZ, MAX_N_LIGHTS)13from .shader_program import ShaderProgramCache14from .material import MetallicRoughnessMaterial, SpecularGlossinessMaterial15from .light import PointLight, SpotLight, DirectionalLight16from .font import FontCache17from .utils import format_color_vector18 19from OpenGL.GL import *20 21 22class Renderer(object):23 """Class for handling all rendering operations on a scene.24 25 Note26 ----27 This renderer relies on the existence of an OpenGL context and28 does not create one on its own.29 30 Parameters31 ----------32 viewport_width : int33 Width of the viewport in pixels.34 viewport_height : int35 Width of the viewport height in pixels.36 point_size : float, optional37 Size of points in pixels. Defaults to 1.0.38 """39 40 def __init__(self, viewport_width, viewport_height, point_size=1.0):41 self.dpscale = 142 # Scaling needed on retina displays43 if sys.platform == 'darwin':44 self.dpscale = 245 46 self.viewport_width = viewport_width47 self.viewport_height = viewport_height48 self.point_size = point_size49 50 # Optional framebuffer for offscreen renders51 self._main_fb = None52 self._main_cb = None53 self._main_db = None54 self._main_fb_ms = None55 self._main_cb_ms = None56 self._main_db_ms = None57 self._main_fb_dims = (None, None)58 self._shadow_fb = None59 self._latest_znear = DEFAULT_Z_NEAR60 self._latest_zfar = DEFAULT_Z_FAR61 62 # Shader Program Cache63 self._program_cache = ShaderProgramCache()64 self._font_cache = FontCache()65 self._meshes = set()66 self._mesh_textures = set()67 self._shadow_textures = set()68 self._texture_alloc_idx = 069 70 @property71 def viewport_width(self):72 """int : The width of the main viewport, in pixels.73 """74 return self._viewport_width75 76 @viewport_width.setter77 def viewport_width(self, value):78 self._viewport_width = self.dpscale * value79 80 @property81 def viewport_height(self):82 """int : The height of the main viewport, in pixels.83 """84 return self._viewport_height85 86 @viewport_height.setter87 def viewport_height(self, value):88 self._viewport_height = self.dpscale * value89 90 @property91 def point_size(self):92 """float : The size of screen-space points, in pixels.93 """94 return self._point_size95 96 @point_size.setter97 def point_size(self, value):98 self._point_size = float(value)99 100 def render(self, scene, flags, seg_node_map=None):101 """Render a scene with the given set of flags.102 103 Parameters104 ----------105 scene : :class:`Scene`106 A scene to render.107 flags : int108 A specification from :class:`.RenderFlags`.109 seg_node_map : dict110 A map from :class:`.Node` objects to (3,) colors for each.111 If specified along with flags set to :attr:`.RenderFlags.SEG`,112 the color image will be a segmentation image.113 114 Returns115 -------116 color_im : (h, w, 3) uint8 or (h, w, 4) uint8117 If :attr:`RenderFlags.OFFSCREEN` is set, the color buffer. This is118 normally an RGB buffer, but if :attr:`.RenderFlags.RGBA` is set,119 the buffer will be a full RGBA buffer.120 depth_im : (h, w) float32121 If :attr:`RenderFlags.OFFSCREEN` is set, the depth buffer122 in linear units.123 """124 # Update context with meshes and textures125 self._update_context(scene, flags)126 127 # Render necessary shadow maps128 if not bool(flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.SEG):129 for ln in scene.light_nodes:130 take_pass = False131 if (isinstance(ln.light, DirectionalLight) and132 bool(flags & RenderFlags.SHADOWS_DIRECTIONAL)):133 take_pass = True134 elif (isinstance(ln.light, SpotLight) and135 bool(flags & RenderFlags.SHADOWS_SPOT)):136 take_pass = True137 elif (isinstance(ln.light, PointLight) and138 bool(flags & RenderFlags.SHADOWS_POINT)):139 take_pass = True140 if take_pass:141 self._shadow_mapping_pass(scene, ln, flags)142 143 # Make forward pass144 retval = self._forward_pass(scene, flags, seg_node_map=seg_node_map)145 146 # If necessary, make normals pass147 if flags & (RenderFlags.VERTEX_NORMALS | RenderFlags.FACE_NORMALS):148 self._normals_pass(scene, flags)149 150 # Update camera settings for retrieving depth buffers151 self._latest_znear = scene.main_camera_node.camera.znear152 self._latest_zfar = scene.main_camera_node.camera.zfar153 154 return retval155 156 def render_text(self, text, x, y, font_name='OpenSans-Regular',157 font_pt=40, color=None, scale=1.0,158 align=TextAlign.BOTTOM_LEFT):159 """Render text into the current viewport.160 161 Note162 ----163 This cannot be done into an offscreen buffer.164 165 Parameters166 ----------167 text : str168 The text to render.169 x : int170 Horizontal pixel location of text.171 y : int172 Vertical pixel location of text.173 font_name : str174 Name of font, from the ``pyrender/fonts`` folder, or175 a path to a ``.ttf`` file.176 font_pt : int177 Height of the text, in font points.178 color : (4,) float179 The color of the text. Default is black.180 scale : int181 Scaling factor for text.182 align : int183 One of the :class:`TextAlign` options which specifies where the184 ``x`` and ``y`` parameters lie on the text. For example,185 :attr:`TextAlign.BOTTOM_LEFT` means that ``x`` and ``y`` indicate186 the position of the bottom-left corner of the textbox.187 """188 x *= self.dpscale189 y *= self.dpscale190 font_pt *= self.dpscale191 192 if color is None:193 color = np.array([0.0, 0.0, 0.0, 1.0])194 else:195 color = format_color_vector(color, 4)196 197 # Set up viewport for render198 self._configure_forward_pass_viewport(0)199 200 # Load font201 font = self._font_cache.get_font(font_name, font_pt)202 if not font._in_context():203 font._add_to_context()204 205 # Load program206 program = self._get_text_program()207 program._bind()208 209 # Set uniforms210 p = np.eye(4)211 p[0,0] = 2.0 / self.viewport_width212 p[0,3] = -1.0213 p[1,1] = 2.0 / self.viewport_height214 p[1,3] = -1.0215 program.set_uniform('projection', p)216 program.set_uniform('text_color', color)217 218 # Draw text219 font.render_string(text, x, y, scale, align)220 221 def read_color_buf(self):222 """Read and return the current viewport's color buffer.223 224 Alpha cannot be computed for an on-screen buffer.225 226 Returns227 -------228 color_im : (h, w, 3) uint8229 The color buffer in RGB byte format.230 """231 # Extract color image from frame buffer232 width, height = self.viewport_width, self.viewport_height233 glBindFramebuffer(GL_READ_FRAMEBUFFER, 0)234 glReadBuffer(GL_FRONT)235 color_buf = glReadPixels(0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE)236 237 # Re-format them into numpy arrays238 color_im = np.frombuffer(color_buf, dtype=np.uint8)239 color_im = color_im.reshape((height, width, 3))240 color_im = np.flip(color_im, axis=0)241 242 # Resize for macos if needed243 if sys.platform == 'darwin':244 color_im = self._resize_image(color_im, True)245 246 return color_im247 248 def read_depth_buf(self):249 """Read and return the current viewport's color buffer.250 251 Returns252 -------253 depth_im : (h, w) float32254 The depth buffer in linear units.255 """256 width, height = self.viewport_width, self.viewport_height257 glBindFramebuffer(GL_READ_FRAMEBUFFER, 0)258 glReadBuffer(GL_FRONT)259 depth_buf = glReadPixels(260 0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT261 )262 263 depth_im = np.frombuffer(depth_buf, dtype=np.float32)264 depth_im = depth_im.reshape((height, width))265 depth_im = np.flip(depth_im, axis=0)266 267 inf_inds = (depth_im == 1.0)268 depth_im = 2.0 * depth_im - 1.0269 z_near, z_far = self._latest_znear, self._latest_zfar270 noninf = np.logical_not(inf_inds)271 if z_far is None:272 depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf])273 else:274 depth_im[noninf] = ((2.0 * z_near * z_far) /275 (z_far + z_near - depth_im[noninf] *276 (z_far - z_near)))277 depth_im[inf_inds] = 0.0278 279 # Resize for macos if needed280 if sys.platform == 'darwin':281 depth_im = self._resize_image(depth_im)282 283 return depth_im284 285 def delete(self):286 """Free all allocated OpenGL resources.287 """288 # Free shaders289 self._program_cache.clear()290 291 # Free fonts292 self._font_cache.clear()293 294 # Free meshes295 for mesh in self._meshes:296 for p in mesh.primitives:297 p.delete()298 299 # Free textures300 for mesh_texture in self._mesh_textures:301 mesh_texture.delete()302 303 for shadow_texture in self._shadow_textures:304 shadow_texture.delete()305 306 self._meshes = set()307 self._mesh_textures = set()308 self._shadow_textures = set()309 self._texture_alloc_idx = 0310 311 self._delete_main_framebuffer()312 self._delete_shadow_framebuffer()313 314 def __del__(self):315 try:316 self.delete()317 except Exception:318 pass319 320 ###########################################################################321 # Rendering passes322 ###########################################################################323 324 def _forward_pass(self, scene, flags, seg_node_map=None):325 # Set up viewport for render326 self._configure_forward_pass_viewport(flags)327 328 # Clear it329 if bool(flags & RenderFlags.SEG):330 glClearColor(0.0, 0.0, 0.0, 1.0)331 if seg_node_map is None:332 seg_node_map = {}333 else:334 glClearColor(*scene.bg_color)335 336 glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)337 338 if not bool(flags & RenderFlags.SEG):339 glEnable(GL_MULTISAMPLE)340 else:341 glDisable(GL_MULTISAMPLE)342 343 # Set up camera matrices344 V, P = self._get_camera_matrices(scene)345 346 program = None347 # Now, render each object in sorted order348 for node in self._sorted_mesh_nodes(scene):349 mesh = node.mesh350 351 # Skip the mesh if it's not visible352 if not mesh.is_visible:353 continue354 355 # If SEG, set color356 if bool(flags & RenderFlags.SEG):357 if node not in seg_node_map:358 continue359 color = seg_node_map[node]360 if not isinstance(color, (list, tuple, np.ndarray)):361 color = np.repeat(color, 3)362 else:363 color = np.asanyarray(color)364 color = color / 255.0365 366 for primitive in mesh.primitives:367 368 # First, get and bind the appropriate program369 program = self._get_primitive_program(370 primitive, flags, ProgramFlags.USE_MATERIAL371 )372 program._bind()373 374 # Set the camera uniforms375 program.set_uniform('V', V)376 program.set_uniform('P', P)377 program.set_uniform(378 'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3]379 )380 if bool(flags & RenderFlags.SEG):381 program.set_uniform('color', color)382 383 # Next, bind the lighting384 if not (flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.FLAT or385 flags & RenderFlags.SEG):386 self._bind_lighting(scene, program, node, flags)387 388 # Finally, bind and draw the primitive389 self._bind_and_draw_primitive(390 primitive=primitive,391 pose=scene.get_pose(node),392 program=program,393 flags=flags394 )395 self._reset_active_textures()396 397 # Unbind the shader and flush the output398 if program is not None:399 program._unbind()400 glFlush()401 402 # If doing offscreen render, copy result from framebuffer and return403 if flags & RenderFlags.OFFSCREEN:404 return self._read_main_framebuffer(scene, flags)405 else:406 return407 408 def _shadow_mapping_pass(self, scene, light_node, flags):409 light = light_node.light410 411 # Set up viewport for render412 self._configure_shadow_mapping_viewport(light, flags)413 414 # Set up camera matrices415 V, P = self._get_light_cam_matrices(scene, light_node, flags)416 417 # Now, render each object in sorted order418 for node in self._sorted_mesh_nodes(scene):419 mesh = node.mesh420 421 # Skip the mesh if it's not visible422 if not mesh.is_visible:423 continue424 425 for primitive in mesh.primitives:426 427 # First, get and bind the appropriate program428 program = self._get_primitive_program(429 primitive, flags, ProgramFlags.NONE430 )431 program._bind()432 433 # Set the camera uniforms434 program.set_uniform('V', V)435 program.set_uniform('P', P)436 program.set_uniform(437 'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3]438 )439 440 # Finally, bind and draw the primitive441 self._bind_and_draw_primitive(442 primitive=primitive,443 pose=scene.get_pose(node),444 program=program,445 flags=RenderFlags.DEPTH_ONLY446 )447 self._reset_active_textures()448 449 # Unbind the shader and flush the output450 if program is not None:451 program._unbind()452 glFlush()453 454 def _normals_pass(self, scene, flags):455 # Set up viewport for render456 self._configure_forward_pass_viewport(flags)457 program = None458 459 # Set up camera matrices460 V, P = self._get_camera_matrices(scene)461 462 # Now, render each object in sorted order463 for node in self._sorted_mesh_nodes(scene):464 mesh = node.mesh465 466 # Skip the mesh if it's not visible467 if not mesh.is_visible:468 continue469 470 for primitive in mesh.primitives:471 472 # Skip objects that don't have normals473 if not primitive.buf_flags & BufFlags.NORMAL:474 continue475 476 # First, get and bind the appropriate program477 pf = ProgramFlags.NONE478 if flags & RenderFlags.VERTEX_NORMALS:479 pf = pf | ProgramFlags.VERTEX_NORMALS480 if flags & RenderFlags.FACE_NORMALS:481 pf = pf | ProgramFlags.FACE_NORMALS482 program = self._get_primitive_program(primitive, flags, pf)483 program._bind()484 485 # Set the camera uniforms486 program.set_uniform('V', V)487 program.set_uniform('P', P)488 program.set_uniform('normal_magnitude', 0.05 * primitive.scale)489 program.set_uniform(490 'normal_color', np.array([0.1, 0.1, 1.0, 1.0])491 )492 493 # Finally, bind and draw the primitive494 self._bind_and_draw_primitive(495 primitive=primitive,496 pose=scene.get_pose(node),497 program=program,498 flags=RenderFlags.DEPTH_ONLY499 )500 self._reset_active_textures()501 502 # Unbind the shader and flush the output503 if program is not None:504 program._unbind()505 glFlush()506 507 ###########################################################################508 # Handlers for binding uniforms and drawing primitives509 ###########################################################################510 511 def _bind_and_draw_primitive(self, primitive, pose, program, flags):512 # Set model pose matrix513 program.set_uniform('M', pose)514 515 # Bind mesh buffers516 primitive._bind()517 518 # Bind mesh material519 if not (flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.SEG):520 material = primitive.material521 522 # Bind textures523 tf = material.tex_flags524 if tf & TexFlags.NORMAL:525 self._bind_texture(material.normalTexture,526 'material.normal_texture', program)527 if tf & TexFlags.OCCLUSION:528 self._bind_texture(material.occlusionTexture,529 'material.occlusion_texture', program)530 if tf & TexFlags.EMISSIVE:531 self._bind_texture(material.emissiveTexture,532 'material.emissive_texture', program)533 if tf & TexFlags.BASE_COLOR:534 self._bind_texture(material.baseColorTexture,535 'material.base_color_texture', program)536 if tf & TexFlags.METALLIC_ROUGHNESS:537 self._bind_texture(material.metallicRoughnessTexture,538 'material.metallic_roughness_texture',539 program)540 if tf & TexFlags.DIFFUSE:541 self._bind_texture(material.diffuseTexture,542 'material.diffuse_texture', program)543 if tf & TexFlags.SPECULAR_GLOSSINESS:544 self._bind_texture(material.specularGlossinessTexture,545 'material.specular_glossiness_texture',546 program)547 548 # Bind other uniforms549 b = 'material.{}'550 program.set_uniform(b.format('emissive_factor'),551 material.emissiveFactor)552 if isinstance(material, MetallicRoughnessMaterial):553 program.set_uniform(b.format('base_color_factor'),554 material.baseColorFactor)555 program.set_uniform(b.format('metallic_factor'),556 material.metallicFactor)557 program.set_uniform(b.format('roughness_factor'),558 material.roughnessFactor)559 elif isinstance(material, SpecularGlossinessMaterial):560 program.set_uniform(b.format('diffuse_factor'),561 material.diffuseFactor)562 program.set_uniform(b.format('specular_factor'),563 material.specularFactor)564 program.set_uniform(b.format('glossiness_factor'),565 material.glossinessFactor)566 567 # Set blending options568 if material.alphaMode == 'BLEND':569 glEnable(GL_BLEND)570 glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)571 else:572 glEnable(GL_BLEND)573 glBlendFunc(GL_ONE, GL_ZERO)574 575 # Set wireframe mode576 wf = material.wireframe577 if flags & RenderFlags.FLIP_WIREFRAME:578 wf = not wf579 if (flags & RenderFlags.ALL_WIREFRAME) or wf:580 glPolygonMode(GL_FRONT_AND_BACK, GL_LINE)581 else:582 glPolygonMode(GL_FRONT_AND_BACK, GL_FILL)583 584 # Set culling mode585 if material.doubleSided or flags & RenderFlags.SKIP_CULL_FACES:586 glDisable(GL_CULL_FACE)587 else:588 glEnable(GL_CULL_FACE)589 glCullFace(GL_BACK)590 else:591 glEnable(GL_CULL_FACE)592 glEnable(GL_BLEND)593 glCullFace(GL_BACK)594 glBlendFunc(GL_ONE, GL_ZERO)595 glPolygonMode(GL_FRONT_AND_BACK, GL_FILL)596 597 # Set point size if needed598 glDisable(GL_PROGRAM_POINT_SIZE)599 if primitive.mode == GLTF.POINTS:600 glEnable(GL_PROGRAM_POINT_SIZE)601 glPointSize(self.point_size)602 603 # Render mesh604 n_instances = 1605 if primitive.poses is not None:606 n_instances = len(primitive.poses)607 608 if primitive.indices is not None:609 glDrawElementsInstanced(610 primitive.mode, primitive.indices.size, GL_UNSIGNED_INT,611 ctypes.c_void_p(0), n_instances612 )613 else:614 glDrawArraysInstanced(615 primitive.mode, 0, len(primitive.positions), n_instances616 )617 618 # Unbind mesh buffers619 primitive._unbind()620 621 def _bind_lighting(self, scene, program, node, flags):622 """Bind all lighting uniform values for a scene.623 """624 max_n_lights = self._compute_max_n_lights(flags)625 626 n_d = min(len(scene.directional_light_nodes), max_n_lights[0])627 n_s = min(len(scene.spot_light_nodes), max_n_lights[1])628 n_p = min(len(scene.point_light_nodes), max_n_lights[2])629 program.set_uniform('ambient_light', scene.ambient_light)630 program.set_uniform('n_directional_lights', n_d)631 program.set_uniform('n_spot_lights', n_s)632 program.set_uniform('n_point_lights', n_p)633 plc = 0634 slc = 0635 dlc = 0636 637 light_nodes = scene.light_nodes638 if (len(scene.directional_light_nodes) > max_n_lights[0] or639 len(scene.spot_light_nodes) > max_n_lights[1] or640 len(scene.point_light_nodes) > max_n_lights[2]):641 light_nodes = self._sorted_nodes_by_distance(642 scene, scene.light_nodes, node643 )644 645 for n in light_nodes:646 light = n.light647 pose = scene.get_pose(n)648 position = pose[:3,3]649 direction = -pose[:3,2]650 651 if isinstance(light, PointLight):652 if plc == max_n_lights[2]:653 continue654 b = 'point_lights[{}].'.format(plc)655 plc += 1656 shadow = bool(flags & RenderFlags.SHADOWS_POINT)657 program.set_uniform(b + 'position', position)658 elif isinstance(light, SpotLight):659 if slc == max_n_lights[1]:660 continue661 b = 'spot_lights[{}].'.format(slc)662 slc += 1663 shadow = bool(flags & RenderFlags.SHADOWS_SPOT)664 las = 1.0 / max(0.001, np.cos(light.innerConeAngle) -665 np.cos(light.outerConeAngle))666 lao = -np.cos(light.outerConeAngle) * las667 program.set_uniform(b + 'direction', direction)668 program.set_uniform(b + 'position', position)669 program.set_uniform(b + 'light_angle_scale', las)670 program.set_uniform(b + 'light_angle_offset', lao)671 else:672 if dlc == max_n_lights[0]:673 continue674 b = 'directional_lights[{}].'.format(dlc)675 dlc += 1676 shadow = bool(flags & RenderFlags.SHADOWS_DIRECTIONAL)677 program.set_uniform(b + 'direction', direction)678 679 program.set_uniform(b + 'color', light.color)680 program.set_uniform(b + 'intensity', light.intensity)681 # if light.range is not None:682 # program.set_uniform(b + 'range', light.range)683 # else:684 # program.set_uniform(b + 'range', 0)685 686 if shadow:687 self._bind_texture(light.shadow_texture,688 b + 'shadow_map', program)689 if not isinstance(light, PointLight):690 V, P = self._get_light_cam_matrices(scene, n, flags)691 program.set_uniform(b + 'light_matrix', P.dot(V))692 else:693 raise NotImplementedError(694 'Point light shadows not implemented'695 )696 697 def _sorted_mesh_nodes(self, scene):698 cam_loc = scene.get_pose(scene.main_camera_node)[:3,3]699 solid_nodes = []700 trans_nodes = []701 for node in scene.mesh_nodes:702 mesh = node.mesh703 if mesh.is_transparent:704 trans_nodes.append(node)705 else:706 solid_nodes.append(node)707 708 # TODO BETTER SORTING METHOD709 trans_nodes.sort(710 key=lambda n: -np.linalg.norm(scene.get_pose(n)[:3,3] - cam_loc)711 )712 solid_nodes.sort(713 key=lambda n: -np.linalg.norm(scene.get_pose(n)[:3,3] - cam_loc)714 )715 716 return solid_nodes + trans_nodes717 718 def _sorted_nodes_by_distance(self, scene, nodes, compare_node):719 nodes = list(nodes)720 compare_posn = scene.get_pose(compare_node)[:3,3]721 nodes.sort(key=lambda n: np.linalg.norm(722 scene.get_pose(n)[:3,3] - compare_posn)723 )724 return nodes725 726 ###########################################################################727 # Context Management728 ###########################################################################729 730 def _update_context(self, scene, flags):731 732 # Update meshes733 scene_meshes = scene.meshes734 735 # Add new meshes to context736 for mesh in scene_meshes - self._meshes:737 for p in mesh.primitives:738 p._add_to_context()739 740 # Remove old meshes from context741 for mesh in self._meshes - scene_meshes:742 for p in mesh.primitives:743 p.delete()744 745 self._meshes = scene_meshes.copy()746 747 # Update mesh textures748 mesh_textures = set()749 for m in scene_meshes:750 for p in m.primitives:751 mesh_textures |= p.material.textures752 753 # Add new textures to context754 for texture in mesh_textures - self._mesh_textures:755 texture._add_to_context()756 757 # Remove old textures from context758 for texture in self._mesh_textures - mesh_textures:759 texture.delete()760 761 self._mesh_textures = mesh_textures.copy()762 763 shadow_textures = set()764 for l in scene.lights:765 # Create if needed766 active = False767 if (isinstance(l, DirectionalLight) and768 flags & RenderFlags.SHADOWS_DIRECTIONAL):769 active = True770 elif (isinstance(l, PointLight) and771 flags & RenderFlags.SHADOWS_POINT):772 active = True773 elif isinstance(l, SpotLight) and flags & RenderFlags.SHADOWS_SPOT:774 active = True775 776 if active and l.shadow_texture is None:777 l._generate_shadow_texture()778 if l.shadow_texture is not None:779 shadow_textures.add(l.shadow_texture)780 781 # Add new textures to context782 for texture in shadow_textures - self._shadow_textures:783 texture._add_to_context()784 785 # Remove old textures from context786 for texture in self._shadow_textures - shadow_textures:787 texture.delete()788 789 self._shadow_textures = shadow_textures.copy()790 791 ###########################################################################792 # Texture Management793 ###########################################################################794 795 def _bind_texture(self, texture, uniform_name, program):796 """Bind a texture to an active texture unit and return797 the texture unit index that was used.798 """799 tex_id = self._get_next_active_texture()800 glActiveTexture(GL_TEXTURE0 + tex_id)801 texture._bind()802 program.set_uniform(uniform_name, tex_id)803 804 def _get_next_active_texture(self):805 val = self._texture_alloc_idx806 self._texture_alloc_idx += 1807 return val808 809 def _reset_active_textures(self):810 self._texture_alloc_idx = 0811 812 ###########################################################################813 # Camera Matrix Management814 ###########################################################################815 816 def _get_camera_matrices(self, scene):817 main_camera_node = scene.main_camera_node818 if main_camera_node is None:819 raise ValueError('Cannot render scene without a camera')820 P = main_camera_node.camera.get_projection_matrix(821 width=self.viewport_width, height=self.viewport_height822 )823 pose = scene.get_pose(main_camera_node)824 V = np.linalg.inv(pose) # V maps from world to camera825 return V, P826 827 def _get_light_cam_matrices(self, scene, light_node, flags):828 light = light_node.light829 pose = scene.get_pose(light_node).copy()830 s = scene.scale831 camera = light._get_shadow_camera(s)832 P = camera.get_projection_matrix()833 if isinstance(light, DirectionalLight):834 direction = -pose[:3,2]835 c = scene.centroid836 loc = c - direction * s837 pose[:3,3] = loc838 V = np.linalg.inv(pose) # V maps from world to camera839 return V, P840 841 ###########################################################################842 # Shader Program Management843 ###########################################################################844 845 def _get_text_program(self):846 program = self._program_cache.get_program(847 vertex_shader='text.vert',848 fragment_shader='text.frag'849 )850 851 if not program._in_context():852 program._add_to_context()853 854 return program855 856 def _compute_max_n_lights(self, flags):857 max_n_lights = [MAX_N_LIGHTS, MAX_N_LIGHTS, MAX_N_LIGHTS]858 n_tex_units = glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS)859 860 # Reserved texture units: 6861 # Normal Map862 # Occlusion Map863 # Emissive Map864 # Base Color or Diffuse Map865 # MR or SG Map866 # Environment cubemap867 868 n_reserved_textures = 6869 n_available_textures = n_tex_units - n_reserved_textures870 871 # Distribute textures evenly among lights with shadows, with872 # a preference for directional lights873 n_shadow_types = 0874 if flags & RenderFlags.SHADOWS_DIRECTIONAL:875 n_shadow_types += 1876 if flags & RenderFlags.SHADOWS_SPOT:877 n_shadow_types += 1878 if flags & RenderFlags.SHADOWS_POINT:879 n_shadow_types += 1880 881 if n_shadow_types > 0:882 tex_per_light = n_available_textures // n_shadow_types883 884 if flags & RenderFlags.SHADOWS_DIRECTIONAL:885 max_n_lights[0] = (886 tex_per_light +887 (n_available_textures - tex_per_light * n_shadow_types)888 )889 if flags & RenderFlags.SHADOWS_SPOT:890 max_n_lights[1] = tex_per_light891 if flags & RenderFlags.SHADOWS_POINT:892 max_n_lights[2] = tex_per_light893 894 return max_n_lights895 896 def _get_primitive_program(self, primitive, flags, program_flags):897 vertex_shader = None898 fragment_shader = None899 geometry_shader = None900 defines = {}901 902 if (bool(program_flags & ProgramFlags.USE_MATERIAL) and903 not flags & RenderFlags.DEPTH_ONLY and904 not flags & RenderFlags.FLAT and905 not flags & RenderFlags.SEG):906 vertex_shader = 'mesh.vert'907 fragment_shader = 'mesh.frag'908 elif bool(program_flags & (ProgramFlags.VERTEX_NORMALS |909 ProgramFlags.FACE_NORMALS)):910 vertex_shader = 'vertex_normals.vert'911 if primitive.mode == GLTF.POINTS:912 geometry_shader = 'vertex_normals_pc.geom'913 else:914 geometry_shader = 'vertex_normals.geom'915 fragment_shader = 'vertex_normals.frag'916 elif flags & RenderFlags.FLAT:917 vertex_shader = 'flat.vert'918 fragment_shader = 'flat.frag'919 elif flags & RenderFlags.SEG:920 vertex_shader = 'segmentation.vert'921 fragment_shader = 'segmentation.frag'922 else:923 vertex_shader = 'mesh_depth.vert'924 fragment_shader = 'mesh_depth.frag'925 926 # Set up vertex buffer DEFINES927 bf = primitive.buf_flags928 buf_idx = 1929 if bf & BufFlags.NORMAL:930 defines['NORMAL_LOC'] = buf_idx931 buf_idx += 1932 if bf & BufFlags.TANGENT:933 defines['TANGENT_LOC'] = buf_idx934 buf_idx += 1935 if bf & BufFlags.TEXCOORD_0:936 defines['TEXCOORD_0_LOC'] = buf_idx937 buf_idx += 1938 if bf & BufFlags.TEXCOORD_1:939 defines['TEXCOORD_1_LOC'] = buf_idx940 buf_idx += 1941 if bf & BufFlags.COLOR_0:942 defines['COLOR_0_LOC'] = buf_idx943 buf_idx += 1944 if bf & BufFlags.JOINTS_0:945 defines['JOINTS_0_LOC'] = buf_idx946 buf_idx += 1947 if bf & BufFlags.WEIGHTS_0:948 defines['WEIGHTS_0_LOC'] = buf_idx949 buf_idx += 1950 defines['INST_M_LOC'] = buf_idx951 952 # Set up shadow mapping defines953 if flags & RenderFlags.SHADOWS_DIRECTIONAL:954 defines['DIRECTIONAL_LIGHT_SHADOWS'] = 1955 if flags & RenderFlags.SHADOWS_SPOT:956 defines['SPOT_LIGHT_SHADOWS'] = 1957 if flags & RenderFlags.SHADOWS_POINT:958 defines['POINT_LIGHT_SHADOWS'] = 1959 max_n_lights = self._compute_max_n_lights(flags)960 defines['MAX_DIRECTIONAL_LIGHTS'] = max_n_lights[0]961 defines['MAX_SPOT_LIGHTS'] = max_n_lights[1]962 defines['MAX_POINT_LIGHTS'] = max_n_lights[2]963 964 # Set up vertex normal defines965 if program_flags & ProgramFlags.VERTEX_NORMALS:966 defines['VERTEX_NORMALS'] = 1967 if program_flags & ProgramFlags.FACE_NORMALS:968 defines['FACE_NORMALS'] = 1969 970 # Set up material texture defines971 if bool(program_flags & ProgramFlags.USE_MATERIAL):972 tf = primitive.material.tex_flags973 if tf & TexFlags.NORMAL:974 defines['HAS_NORMAL_TEX'] = 1975 if tf & TexFlags.OCCLUSION:976 defines['HAS_OCCLUSION_TEX'] = 1977 if tf & TexFlags.EMISSIVE:978 defines['HAS_EMISSIVE_TEX'] = 1979 if tf & TexFlags.BASE_COLOR:980 defines['HAS_BASE_COLOR_TEX'] = 1981 if tf & TexFlags.METALLIC_ROUGHNESS:982 defines['HAS_METALLIC_ROUGHNESS_TEX'] = 1983 if tf & TexFlags.DIFFUSE:984 defines['HAS_DIFFUSE_TEX'] = 1985 if tf & TexFlags.SPECULAR_GLOSSINESS:986 defines['HAS_SPECULAR_GLOSSINESS_TEX'] = 1987 if isinstance(primitive.material, MetallicRoughnessMaterial):988 defines['USE_METALLIC_MATERIAL'] = 1989 elif isinstance(primitive.material, SpecularGlossinessMaterial):990 defines['USE_GLOSSY_MATERIAL'] = 1991 992 program = self._program_cache.get_program(993 vertex_shader=vertex_shader,994 fragment_shader=fragment_shader,995 geometry_shader=geometry_shader,996 defines=defines997 )998 999 if not program._in_context():1000 program._add_to_context()1001 1002 return program1003 1004 ###########################################################################1005 # Viewport Management1006 ###########################################################################1007 1008 def _configure_forward_pass_viewport(self, flags):1009 1010 # If using offscreen render, bind main framebuffer1011 if flags & RenderFlags.OFFSCREEN:1012 self._configure_main_framebuffer()1013 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb_ms)1014 else:1015 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0)1016 1017 glViewport(0, 0, self.viewport_width, self.viewport_height)1018 glEnable(GL_DEPTH_TEST)1019 glDepthMask(GL_TRUE)1020 glDepthFunc(GL_LESS)1021 glDepthRange(0.0, 1.0)1022 1023 def _configure_shadow_mapping_viewport(self, light, flags):1024 self._configure_shadow_framebuffer()1025 glBindFramebuffer(GL_FRAMEBUFFER, self._shadow_fb)1026 light.shadow_texture._bind()1027 light.shadow_texture._bind_as_depth_attachment()1028 glActiveTexture(GL_TEXTURE0)1029 light.shadow_texture._bind()1030 glDrawBuffer(GL_NONE)1031 glReadBuffer(GL_NONE)1032 1033 glClear(GL_DEPTH_BUFFER_BIT)1034 glViewport(0, 0, SHADOW_TEX_SZ, SHADOW_TEX_SZ)1035 glEnable(GL_DEPTH_TEST)1036 glDepthMask(GL_TRUE)1037 glDepthFunc(GL_LESS)1038 glDepthRange(0.0, 1.0)1039 glDisable(GL_CULL_FACE)1040 glDisable(GL_BLEND)1041 1042 ###########################################################################1043 # Framebuffer Management1044 ###########################################################################1045 1046 def _configure_shadow_framebuffer(self):1047 if self._shadow_fb is None:1048 self._shadow_fb = glGenFramebuffers(1)1049 1050 def _delete_shadow_framebuffer(self):1051 if self._shadow_fb is not None:1052 glDeleteFramebuffers(1, [self._shadow_fb])1053 1054 def _configure_main_framebuffer(self):1055 # If mismatch with prior framebuffer, delete it1056 if (self._main_fb is not None and1057 self.viewport_width != self._main_fb_dims[0] or1058 self.viewport_height != self._main_fb_dims[1]):1059 self._delete_main_framebuffer()1060 1061 # If framebuffer doesn't exist, create it1062 if self._main_fb is None:1063 # Generate standard buffer1064 self._main_cb, self._main_db = glGenRenderbuffers(2)1065 1066 glBindRenderbuffer(GL_RENDERBUFFER, self._main_cb)1067 glRenderbufferStorage(1068 GL_RENDERBUFFER, GL_RGBA,1069 self.viewport_width, self.viewport_height1070 )1071 1072 glBindRenderbuffer(GL_RENDERBUFFER, self._main_db)1073 glRenderbufferStorage(1074 GL_RENDERBUFFER, GL_DEPTH_COMPONENT24,1075 self.viewport_width, self.viewport_height1076 )1077 1078 self._main_fb = glGenFramebuffers(1)1079 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb)1080 glFramebufferRenderbuffer(1081 GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,1082 GL_RENDERBUFFER, self._main_cb1083 )1084 glFramebufferRenderbuffer(1085 GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,1086 GL_RENDERBUFFER, self._main_db1087 )1088 1089 # Generate multisample buffer1090 self._main_cb_ms, self._main_db_ms = glGenRenderbuffers(2)1091 glBindRenderbuffer(GL_RENDERBUFFER, self._main_cb_ms)1092 # glRenderbufferStorageMultisample(1093 # GL_RENDERBUFFER, 4, GL_RGBA,1094 # self.viewport_width, self.viewport_height1095 # )1096 # glBindRenderbuffer(GL_RENDERBUFFER, self._main_db_ms)1097 # glRenderbufferStorageMultisample(1098 # GL_RENDERBUFFER, 4, GL_DEPTH_COMPONENT24,1099 # self.viewport_width, self.viewport_height1100 # )1101 # 增加这一行1102 num_samples = min(glGetIntegerv(GL_MAX_SAMPLES), 4) # No more than GL_MAX_SAMPLES1103 1104 # 其实就是把 4 替换成 num_samples，其余不变1105 glRenderbufferStorageMultisample(GL_RENDERBUFFER, num_samples, GL_RGBA, self.viewport_width, self.viewport_height)1106 1107 glBindRenderbuffer(GL_RENDERBUFFER, self._main_db_ms) # 这行不变1108 1109 # 这一行也是将 4 替换成 num_samples1110 glRenderbufferStorageMultisample(GL_RENDERBUFFER, num_samples, GL_DEPTH_COMPONENT24, self.viewport_width, self.viewport_height)1111 1112 self._main_fb_ms = glGenFramebuffers(1)1113 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb_ms)1114 glFramebufferRenderbuffer(1115 GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,1116 GL_RENDERBUFFER, self._main_cb_ms1117 )1118 glFramebufferRenderbuffer(1119 GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,1120 GL_RENDERBUFFER, self._main_db_ms1121 )1122 1123 self._main_fb_dims = (self.viewport_width, self.viewport_height)1124 1125 def _delete_main_framebuffer(self):1126 if self._main_fb is not None:1127 glDeleteFramebuffers(2, [self._main_fb, self._main_fb_ms])1128 if self._main_cb is not None:1129 glDeleteRenderbuffers(2, [self._main_cb, self._main_cb_ms])1130 if self._main_db is not None:1131 glDeleteRenderbuffers(2, [self._main_db, self._main_db_ms])1132 1133 self._main_fb = None1134 self._main_cb = None1135 self._main_db = None1136 self._main_fb_ms = None1137 self._main_cb_ms = None1138 self._main_db_ms = None1139 self._main_fb_dims = (None, None)1140 1141 def _read_main_framebuffer(self, scene, flags):1142 width, height = self._main_fb_dims[0], self._main_fb_dims[1]1143 1144 # Bind framebuffer and blit buffers1145 glBindFramebuffer(GL_READ_FRAMEBUFFER, self._main_fb_ms)1146 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb)1147 glBlitFramebuffer(1148 0, 0, width, height, 0, 0, width, height,1149 GL_COLOR_BUFFER_BIT, GL_LINEAR1150 )1151 glBlitFramebuffer(1152 0, 0, width, height, 0, 0, width, height,1153 GL_DEPTH_BUFFER_BIT, GL_NEAREST1154 )1155 glBindFramebuffer(GL_READ_FRAMEBUFFER, self._main_fb)1156 1157 # Read depth1158 depth_buf = glReadPixels(1159 0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT1160 )1161 depth_im = np.frombuffer(depth_buf, dtype=np.float32)1162 depth_im = depth_im.reshape((height, width))1163 depth_im = np.flip(depth_im, axis=0)1164 inf_inds = (depth_im == 1.0)1165 depth_im = 2.0 * depth_im - 1.01166 z_near = scene.main_camera_node.camera.znear1167 z_far = scene.main_camera_node.camera.zfar1168 noninf = np.logical_not(inf_inds)1169 if z_far is None:1170 depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf])1171 else:1172 depth_im[noninf] = ((2.0 * z_near * z_far) /1173 (z_far + z_near - depth_im[noninf] *1174 (z_far - z_near)))1175 depth_im[inf_inds] = 0.01176 1177 # Resize for macos if needed1178 if sys.platform == 'darwin':1179 depth_im = self._resize_image(depth_im)1180 1181 if flags & RenderFlags.DEPTH_ONLY:1182 return depth_im1183 1184 # Read color1185 if flags & RenderFlags.RGBA:1186 color_buf = glReadPixels(1187 0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE1188 )1189 color_im = np.frombuffer(color_buf, dtype=np.uint8)1190 color_im = color_im.reshape((height, width, 4))1191 else:1192 color_buf = glReadPixels(1193 0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE1194 )1195 color_im = np.frombuffer(color_buf, dtype=np.uint8)1196 color_im = color_im.reshape((height, width, 3))1197 color_im = np.flip(color_im, axis=0)1198 1199 # Resize for macos if needed1200 if sys.platform == 'darwin':