Coverage for dragonfly_sim/core/windtunnel_model.py: 85%
560 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-21 00:13 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-21 00:13 +0000
1import numpy as np
2from mpi4py import MPI
3from petsc4py import PETSc
4import dolfinx
6import datetime
7import resource
8import os
9import gc
11import csdl_alpha as csdl
13from time import perf_counter
15from dragonfly_sim.core.mesh_manager import Mesh
16from dragonfly_sim.core.Euler_model import CompressibleEulerModel
17from dragonfly_sim.core.meshwarping import IDWarp_jax
18from dragonfly_sim.utils.data_handler import DataStore
20from dragonfly_sim.utils.petsc_utils import set_petsc_vec_array
21from dragonfly_sim.utils.derived_fields import destroy_chained_operator
22from dragonfly_sim.utils.solver_utils import ksp_converged_reason_name
23from dragonfly_sim.core.postprocessor import DG_postprocessor
26# TODO: Look into using entropy variables instead of conservative variables;
27# these would potentially allow us to do simulations from low Mach all
28# the way to supersonic
29class DG_windtunnel_model(csdl.experimental.CustomImplicitOperation):
30 # The solve/adjoint callbacks contain MPI collectives (PETSc, mpi4py) and deform
31 # the shared mesh, so they must never overlap -- also on a single process.
32 ordered_callbacks = True
34 def __init__(self, mesh, boundary_dict, shape_parameterization=None, moments=None,
35 mesh_inner_bdry_function=None, wall_tags=None, symmetry_tags=None,
36 poly_order=1, gamma=1.4,
37 filename_suffix="test",
38 asm_overlap=None, ilu_levels=None,
39 model_class=CompressibleEulerModel, model_kwargs=None,
40 reduced_order_model=None, compute_all_adjoint_errors=False):
41 """
42 boundary_dict: the free-stream state {'rho', 'p', 'M', 'alpha'} (density,
43 pressure, Mach number, angle of attack in rad). The whole outer boundary
44 is a characteristic far field built from it, so the angle of attack can
45 change by any amount during an optimization; there is no inflow/outflow
46 division to choose or to keep valid.
48 The shape of the object being optimized (the wall) is defined in one of
49 three ways, in this order of precedence:
50 1. `mesh_inner_bdry_function`, a function of the boundary facet midpoints
51 (shape (dim, n)) returning a boolean wall mask. It can encode
52 exclusions the FFD block cannot. If the mesh also carries boundary
53 facet tags (utils/mesh_io_utils.py:load_dolfinx_mesh stores them on
54 the mesh), a warning lists them: they could be used instead.
55 2. `wall_tags`: the mesh's boundary facet tags (values or names, one or
56 several) that are the wall, and `symmetry_tags` those that are
57 symmetry planes (slip walls). Every other boundary is far field.
58 In 3D without tags, the symmetry plane is y = 0.
59 3. The shape parameterization's FFD block corners
60 (shape_parameterization.inner_boundary_function()), which bound the
61 box the inner boundary facets are contained in.
62 At least one is required.
64 shape_parameterization: the wall-shape parameterization, e.g. a
65 shape_design.FFDShapeParameterization -- the FFD block and the layers
66 that define the shape design variables and their bounds. The model does
67 not need to know what those variables are: set_up_sim builds the IDWarp
68 mesh warper (self.mesh_warper) and hands the wall to
69 shape_parameterization.setup(); deform_mesh() turns its wall
70 displacement into mesh node motions; evaluate() takes those and the
71 flat shape parameter vector: shape optimization. Without it (None) no
72 warper is built, the mesh stays fixed, and evaluate() takes alpha only:
73 a forward flow analysis, with derivatives with respect to alpha.
74 solve_forward() runs one flow solve without a CSDL graph at all.
76 moments: the moments of the wall force to compute, print and differentiate,
77 a sequence of (name, point, direction[, ref_length]) tuples (see
78 DG_postprocessor); default none. E.g. the pitching moment about the quarter
79 chord of a wing, ("pitch", (0.25, 0., 0.), "pitch"), and its root
80 bending moment, ("root_bending", (0.25, 0., 0.), (1., 0., 0.)).
82 model_class / model_kwargs select the flow model: CompressibleEulerModel
83 (the default) or e.g. core.RANS_model.CompressibleRANSModel with
84 model_kwargs={'Re': 6e6}. The body's wall is the model's
85 define_wall_bc (slip for Euler, adiabatic no-slip for RANS).
87 reduced_order_model: a core.reduced_order_model.ReducedOrderModel to
88 try before every flow solve (see _solve), or None for full-order
89 solves only. Where its solution is accepted, the derivatives use the
90 adjoint of its projected equations (see apply_inverse_jacobian).
92 compute_all_adjoint_errors: diagnostic, default False. When a derivative
93 uses the ROM adjoint, also solve the full-order adjoint at the same state
94 and right-hand side and record the ROM adjoint's error against it in
95 data_store.adjoint_errors. The adjoint handed to the optimizer is the
96 ROM's either way; the extra full-order solve costs about as much as the
97 adjoint the ROM saves, and is not part of data_store.adjoint_solves.
98 """
99 super().__init__()
101 self.asm_overlap = asm_overlap
102 self.ilu_levels = ilu_levels
104 # --- Linear-solve convergence criteria ----------------------------
105 # We use one set of convergence criteria for all linear solves.
106 # This does not cover the Newton solver for the Euler model itself,
107 # but all other things (adjoint, Helmholtz, elasticity-based
108 # mesh warping, etc.) when used.
109 self.linear_solver_rtol = 1e-20
110 self.linear_solver_atol = 1e-11
111 self.linear_solver_max_it = 1000
113 # If linear solves converge above this tolerance, we raise an error
114 self.linear_residual_accept_limit = 1e-10
116 # --- Nonlinear (Euler) convergence criteria -----------------------
117 self.euler_residual_conv_limit = 1e-9
118 # Total Newton-step budget for the FOM solve
119 self.max_newton_iterations = 100
120 # KDTree coincidence radius for the dof <-> mesh-node permutation.
121 # These two orderings are 1:1 by construction and differ only in
122 # index order, so this tolerance accounts for round-off errors
123 self.mesh_node_match_tol = 1e-10
125 if shape_parameterization is not None and not hasattr(shape_parameterization, 'setup'):
126 raise TypeError("shape_parameterization must be a shape parameterization object "
127 "(e.g. shape_design.FFDShapeParameterization), got {!r}. The FFD "
128 "block arguments (ffd_shape, ffd_degree, ffd_block_corner_list, "
129 "cp_coord_opt_idxs) moved there.".format(shape_parameterization))
130 if mesh_inner_bdry_function is None and wall_tags is None and shape_parameterization is not None:
131 mesh_inner_bdry_function = shape_parameterization.inner_boundary_function()
132 if mesh_inner_bdry_function is None and wall_tags is None:
133 raise ValueError("Provide either mesh_inner_bdry_function, wall_tags (the mesh's own "
134 "boundary tags) or a shape parameterization with FFD block corners "
135 "to locate the inner boundary facets.")
137 # Initial instantiation of mesh and simulation model
138 self.mesh = Mesh(mesh, mesh_inner_bdry_function,
139 node_match_tol=self.mesh_node_match_tol,
140 wall_tags=wall_tags, symmetry_tags=symmetry_tags)
141 if model_class is CompressibleEulerModel:
142 _euler_kwargs = dict(poly_order=poly_order, gamma=gamma)
143 self.sim_model = CompressibleEulerModel(self.mesh, **_euler_kwargs)
144 else:
145 self.sim_model = model_class(self.mesh, gamma=gamma, **(model_kwargs or {}))
146 # TODO: Perhaps split up CompressibleEulerModel into an Euler model class and a solver class that facilitates solver interactions;
147 # the solver has become so complicated recently that spinning it off into its own class would likely help with code readability
149 if asm_overlap is not None:
150 self.sim_model.asm_overlap = asm_overlap
151 if ilu_levels is not None:
152 self.sim_model.ilu_levels = ilu_levels
154 # Set by solve_residual_equations, once, as the LAST write to this
155 # name for the evaluation: whether the solve that produced the
156 # accepted u_vec actually converged.
157 # Mainly used for testing purposes.
158 self.sim_model.last_solve_converged = False
160 # Mesh warping only with a shape parameterization (see __init__)
161 self.shape_parameterization = shape_parameterization
162 self.shape_parameterized = shape_parameterization is not None
164 # The mesh warper, built in set_up_sim()
165 self.mesh_warper = None
167 # Define data storage object for external post-processing
168 self.data_store = DataStore()
170 self.previous_solution = None
172 # POD reduced-order model (None: full-order solves only)
173 self.reduced_order_model = reduced_order_model
174 self.compute_all_adjoint_errors = compute_all_adjoint_errors
175 # (eval_idx, number of adjoint solves so far at it): the output row of
176 # the derivative evaluation that CSDL is on
177 self._adjoint_row = (None, 0)
178 # (parameter vector, converged) of the last evaluation if the FOM
179 # solved it, else None: a re-evaluation of that design skips the ROM
180 # (see _solve)
181 self._fom_design = None
183 # Naming suffix for exported files
184 self.filename_suffix = filename_suffix
186 # Free the flow solver's matrices after every forward solve and the
187 # condensed dR/dU after every condensed adjoint
188 # (sim_model.release_solver), so the two never coexist; they are
189 # rebuilt when next needed. For memory-bound runs (3D RANS).
190 self.release_solver_memory = False
192 # Adjoint reuse. The transposed dR/dU and the preconditioned KSP of the
193 # state last linearized (condensed Jacobian), kept for the adjoint
194 # solves of every output of that design: {'key': state key, 'ksp': KSP};
195 # freed when the next flow solve starts or a different state is
196 # linearized.
197 self._adjoint_cache = None
199 # Options of the adjoint of a model with a condensed Jacobian (default off;
200 # set_adjoint_options sets them together with those of the reconstruction):
201 # adjoint_matrix_free the adjoint operator A_UU^T x + B^T (A_UG^T x), without
202 # forming A or the product A_UG B
203 # cache_jacvec_operators the dR/dx operator and dR/dalpha of the
204 # compute_jacvec_product calls of all outputs of one
205 # design are built once (freed with the adjoint cache)
206 self.adjoint_matrix_free = False
207 self.cache_jacvec_operators = False
208 # PETSc options (names without the "adj_" prefix) overriding the adjoint KSP's settings of the
209 # model, e.g. {"ksp_type": "hpddm", "ksp_hpddm_type": "gcrodr", "ksp_hpddm_recycle": 20}; the KSP
210 # is kept for all outputs of a design, so a recycling method carries its subspace between them
211 self.adjoint_ksp_options = None
212 self._dRdx_cache = None
213 self._dRdalpha_cache = None
215 # Start each condensed adjoint solve from a first-order solution: the
216 # solution of the same transposed system with the compact (nearest
217 # neighbour, gradients frozen) part A_UU of the Jacobian alone -- the
218 # first-order Jacobian the preconditioner is built from -- solved to a
219 # relative tolerance first_order_adjoint_rtol. It reuses the
220 # preconditioner's factors, so costs only Krylov iterations. Applies to
221 # models with a condensed Jacobian (MUSCL / RANS); the guess is always
222 # used, and its effect is logged (self._last_first_order_guess).
223 self.adjoint_first_order_guess = False
224 self.first_order_adjoint_rtol = 1e-2
225 self._last_first_order_guess = None
227 # Write mesh_deformation_<suffix>.bp (the node motions of every design
228 # evaluation). The solution files are written either way.
229 self.write_mesh_deformation = True
231 # Write the output frame (solution and mesh deformation) only for the
232 # first evaluation of each design: an optimizer that evaluates the
233 # same design again (CSDL re-runs the model for the derivatives) then
234 # does not duplicate the frame. Frames keep the eval_idx of that first
235 # evaluation, so their labels have gaps.
236 self.write_once_per_design = False
237 self._last_written_design = None
240 # Discretization and physics-related parameters
241 self.poly_order:int = poly_order
242 self.gamma:float = gamma
243 self.boundary_condition_dict = boundary_dict
244 self.moments = moments
246 # Dolfinx mesh tags for the various boundaries
247 self.WALL_TAG = 2 # walls (including wing/airfoil boundaries)
248 self.SYMMETRY_TAG = 5 # symmetry planes (3D: y = 0, or the mesh's symmetry_tags)
249 self.FARFIELD_TAG = 6 # characteristic far field: the whole outer boundary
251 # global evaluation index of solve_residual_equations
252 self.eval_idx = 0
254 def set_up_sim(self):
255 # pass the various tolerances and limits to the Euler sim model
256 self.sim_model.residual_conv_limit = self.euler_residual_conv_limit
257 self.sim_model.max_newton_iterations = self.max_newton_iterations
259 # use the free-stream dictionary to define the far-field conditions
260 self.sim_model.define_farfield_conditions(self.boundary_condition_dict)
262 # compute boundary facet boolean arrays
263 inner_facets = self.mesh.inner_bdry_facet_mask
265 # Find the symmetry face facets (only relevant in 3D at the moment)
266 # NOTE: The symmetry plane is currently hardcoded at y=0
267 has_symmetry = self.mesh.symmetry_facet_mask is not None or self.sim_model.dimensions == 3
268 if self.mesh.symmetry_facet_mask is not None:
269 symmetry_facets = self.mesh.symmetry_facet_mask # from the mesh's boundary tags
270 elif self.sim_model.dimensions == 3:
271 symmetry_facets = self.mesh.bdry_midpoints[:, 1] <= 1e-6
272 else:
273 symmetry_facets = np.array([False]*inner_facets.shape[0])
275 # Everything that is neither wall nor symmetry plane is far field. The
276 # characteristic far-field state decides facet by facet whether the flow
277 # enters or leaves, so no inflow/outflow division (and nothing to
278 # re-tag when the angle of attack changes) is needed.
279 farfield_facets = ~np.asarray(inner_facets, dtype=bool) & ~np.asarray(symmetry_facets, dtype=bool)
281 # Print the local number of facets of each type for each MPI rank
282 PETSc.Sys.syncPrint("Rank {}, number of wing boundary facets: {}".format(self.mesh.mesh.comm.Get_rank(), np.count_nonzero(inner_facets)))
283 PETSc.Sys.syncPrint("Rank {}, number of symmetry facets: {}".format(self.mesh.mesh.comm.Get_rank(), np.count_nonzero(symmetry_facets)))
284 PETSc.Sys.syncPrint("Rank {}, number of far-field facets: {}".format(self.mesh.mesh.comm.Get_rank(), np.count_nonzero(farfield_facets)))
285 PETSc.Sys.syncFlush()
287 # Pair the boundary facet sets with the various mesh tags
288 bdry_meshtags_and_ds_list = [(farfield_facets, self.FARFIELD_TAG), (inner_facets, self.WALL_TAG)]
289 if has_symmetry:
290 bdry_meshtags_and_ds_list += [(symmetry_facets, self.SYMMETRY_TAG)]
292 # Build the boundary facet tags and define the corresponding quadrature measure
293 self.mesh.tag_boundary_facets(bdry_meshtags_and_ds_list)
294 self.mesh.form_manager.build_boundary_measure(
295 measure_metadata={'quadrature_degree': 3*self.poly_order + 1})
297 self.mesh.configure_output(self.filename_suffix,
298 deformation=self.shape_parameterized and self.write_mesh_deformation)
300 if self.shape_parameterized:
301 # The JAX-based IDWarp implementation as mesh warper
302 self.mesh_warper = IDWarp_jax(self.mesh, wall_meshtag_val=self.WALL_TAG,
303 anchor_meshtag_vals=(self.FARFIELD_TAG,))
305 PETSc.Sys.Print("Initialized mesh warping object")
307 # The shape parameterization gets the wall nodes (replicated, in the
308 # numbering of the mesh's wall node set) and facets; its
309 # wall_displacement() is the (M_global, dim) array mesh_warper.evaluate
310 # takes.
311 wall_nodes = self.mesh.boundary_node_set((self.WALL_TAG,))
312 self.shape_parameterization.setup(self.mesh.mesh.comm, wall_nodes.coords,
313 wall_nodes.global_idx,
314 self.mesh.wall_facets(self.WALL_TAG))
315 else:
316 PETSc.Sys.Print("No shape parameterization given: fixed mesh, forward flow analysis")
318 # define the integration measure for the mesh cells and non-boundary facets
319 self.mesh.form_manager.build_cell_and_interior_facet_measures(
320 measure_metadata={'quadrature_degree': 3*self.poly_order + 1})
322 # Define the required Dolfinx objects for the simulation, including the initial conditions
323 self.sim_model.define_elements_functionspaces()
324 self.sim_model.define_trial_testfunctions()
325 self.sim_model.compute_initial_conditions_from_farfield()
326 self.sim_model.interpolate_solution_vector()
328 # We output the facet mesh tags to an xdmf file, so we can inspect the BCs in paraview
329 self.mesh.mesh.topology.create_connectivity(self.sim_model.dimensions-1, self.sim_model.dimensions)
330 with dolfinx.io.XDMFFile(self.mesh.mesh.comm, "output_meshtags.xdmf", "w") as xdmf:
331 # Write the Mesh object to the file
332 xdmf.write_mesh(self.mesh.mesh)
333 # Write the MeshTags object to the file
334 xdmf.write_meshtags(self.mesh.meshtags, self.mesh.mesh.geometry)
336 # Define the boundary condition terms of the Euler weak form
337 self.sim_model.define_farfield_bc(self.FARFIELD_TAG)
338 self.sim_model.define_wall_bc(self.WALL_TAG)
339 if has_symmetry:
340 self.sim_model.define_slipwall_bc(self.SYMMETRY_TAG)
342 # Construct the Euler weak form for forward evaluation
343 self.sim_model.compute_weakform()
345 # Define the file names and objects for solution writes to external files for Paraview
346 self.sim_model.define_sol_export_files(self.sim_model.functionspaces["V"],
347 file_name_addendum=self.filename_suffix,
348 mesh_deformation=self.shape_parameterized)
350 # We instantiate a separate DG_postprocessor object here so we can print
351 # forces and moments and coefficients after each forward solve
352 self.postprocessor = DG_postprocessor(
353 self.mesh, self.sim_model, self.WALL_TAG,
354 moments=self.moments,
355 p_inf_dim=101325.)
357 if self.reduced_order_model is not None:
358 self.reduced_order_model.set_up(self.sim_model)
360 def _log_memory(self, stage):
361 # We write a log file for each MPI rank that documents the peak local memory usage
362 # at various points during an optimization run.
363 mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
364 rank = MPI.COMM_WORLD.Get_rank()
365 fname = 'memory_logs/r{}_{}.log'.format(rank, self.filename_suffix)
366 os.makedirs('memory_logs', exist_ok=True)
367 with open(fname, 'a') as f:
368 f.write('{} {} {} {} {}\n'.format(
369 self.eval_idx, datetime.datetime.now(), os.getpid(), mem, stage))
371 @property
372 def ffd(self):
373 """The shape parameterization's FFD block (None without one, or before
374 set_up_sim)."""
375 if self.shape_parameterization is None:
376 return None
377 return getattr(self.shape_parameterization, 'ffd', None)
379 def wall_geometry(self):
380 """Geometric constraint quantities of the FFD-deformed wall (area/volume,
381 thickness, planform); the shape parameterization's
382 shape_parameterization.WallGeometry.
384 Built on first use. Collective: call it on every rank.
385 """
386 if self.ffd is None:
387 raise RuntimeError("wall_geometry needs a shape parameterization (an FFD block), "
388 "set up by set_up_sim")
389 return self.shape_parameterization.geometry
391 def deform_mesh(self):
392 """Mesh node motions (CSDL) for the shape parameterization's current
393 wall displacement: the first input of evaluate(). Call after the
394 shape design variables are declared."""
395 if not self.shape_parameterized:
396 raise RuntimeError("deform_mesh needs a shape parameterization")
397 shape = self.shape_parameterization
398 return self.mesh_warper.evaluate(shape.wall_displacement(), shape.parameter_vector())
400 def evaluate(self, mesh_node_motions:csdl.Variable = None, shape_parameters:csdl.Variable = None,
401 alpha:csdl.Variable = None):
402 """
403 The flow state u_vec as a CSDL output.
405 Shape optimization (a shape parameterization was given):
406 evaluate(mesh_node_motions, shape_parameters, alpha), with
407 mesh_node_motions from deform_mesh() and shape_parameters the flat
408 shape parameter vector. Forward analysis on the fixed mesh:
409 evaluate(alpha=alpha). alpha is always required.
410 """
411 if alpha is None:
412 raise ValueError("evaluate needs alpha")
413 if (mesh_node_motions is None) != (shape_parameters is None):
414 raise ValueError("pass mesh_node_motions and shape_parameters together, or neither")
415 if mesh_node_motions is not None and not self.shape_parameterized:
416 raise ValueError("mesh_node_motions given, but there is no shape parameterization / "
417 "mesh warper (pass shape_parameterization to DG_windtunnel_model)")
419 # set inputs using self.declare_input
420 if mesh_node_motions is not None:
421 self.declare_input('mesh_node_motions', mesh_node_motions)
422 self.declare_input('shape_parameters', shape_parameters)
423 self.declare_input('alpha', alpha)
425 # The output on this rank is only the local portion of the solution vector
426 u_vec = self.create_output('u_vec', (self.sim_model.u_vec.x.petsc_vec.local_size,))
428 if mesh_node_motions is not None:
429 # We have declared shape_parameters as an input here, just for logging and computation purposes,
430 # not because we actually need it; mesh_node_motions carries the effect of the shape_parameters.
431 self.declare_derivative_parameters(of='u_vec', wrt='shape_parameters', dependent=False)
433 return u_vec
435 def solve_forward(self, alpha=None, snapshot=True):
436 """
437 One flow solve on the current (undeformed) mesh, without CSDL: set
438 alpha (radians; default: the boundary dictionary's), solve, print the
439 coefficients. Warm-starts from the previous solve. Call set_up_sim()
440 first. Returns (u_vec array, converged, forces), forces being
441 DG_postprocessor.forces_and_coefficients (L, D, c_l, c_d, the moments, and
442 the friction drag for a viscous model).
444 The ROM (if any) stores the solve as a snapshot at the parameter vector
445 (alpha). With shape parameters that vector is longer, so a solve for a
446 baseline that is not a design of the optimization passes snapshot=False.
447 """
448 if alpha is not None:
449 self.sim_model.set_angle_of_attack(alpha)
450 self.postprocessor.set_angle_of_attack(alpha)
451 self.sim_model.compute_weakform()
452 rom, self.reduced_order_model = self.reduced_order_model, (self.reduced_order_model if snapshot else None)
453 try:
454 solution, converged = self._solve(
455 self._parameter_vector(None, self.sim_model.farfield_conditions['alpha']))
456 finally:
457 self.reduced_order_model = rom
458 self.sim_model.last_solve_converged = converged
459 if np.isfinite(np.linalg.norm(solution)):
460 self.previous_solution = solution
461 return solution, converged, self.postprocessor.forces_and_coefficients(solution)
463 def solve_residual_equations(self, input_vals, output_vals):
464 # Define inputs
465 mesh_node_motions = input_vals['mesh_node_motions'] if 'mesh_node_motions' in input_vals else None
466 shape_parameters = input_vals['shape_parameters'] if 'shape_parameters' in input_vals else None
467 alpha = input_vals['alpha'][0]
469 # Update the angle of attack to its current value
470 self.sim_model.set_angle_of_attack(alpha)
472 PETSc.Sys.Print("angle of attack: {} deg".format(np.degrees(alpha)))
473 PETSc.Sys.Print("shape parameters: {}".format(shape_parameters))
474 PETSc.Sys.Print("solve_residual_equations, stepping into mesh.apply_node_motions")
475 if mesh_node_motions is not None:
476 PETSc.Sys.Print("mesh_node_motions 2-norm: {}".format(np.linalg.norm(mesh_node_motions)))
478 # Apply the mesh node motions to the mesh
479 self.mesh.apply_node_motions(mesh_node_motions, design=shape_parameters)
481 # If the mesh is not valid (e.g. has inverted cells), set the output to zero and don't run any simulations
482 if not self.mesh.is_valid:
483 PETSc.Sys.Print("Mesh is not valid, skipping simulations.")
484 output_vals['u_vec'] = np.zeros((self.sim_model.u_vec.x.petsc_vec.local_size,))
485 self._fom_design = None
486 if self.reduced_order_model is not None:
487 self.reduced_order_model.discard_adjoint()
488 return
490 # re-define the current weak form if necessary
491 self.sim_model.compute_weakform()
493 # --- flow solve (ROM, else FOM) -----------------------------------
494 solution_output, solution_converged = self._solve(self._parameter_vector(shape_parameters, alpha))
496 self._log_memory("after flow solve, before setting u_vec output")
498 self.sim_model.last_solve_converged = solution_converged
500 PETSc.Sys.Print("setting u_vec output...")
501 output_vals['u_vec'] = solution_output
503 # Warm start: keep the accepted solution unless it has blown up
504 if np.isfinite(np.linalg.norm(solution_output)):
505 PETSc.Sys.Print("Updating warm start solution")
506 self.previous_solution = solution_output
507 else:
508 PETSc.Sys.Print("Not updating warm start solution, using previous value")
510 design = np.concatenate([[alpha], np.ravel(shape_parameters)
511 if shape_parameters is not None else []])
512 if (self.write_once_per_design and self._last_written_design is not None
513 and np.array_equal(design, self._last_written_design)):
514 PETSc.Sys.Print("Same design as the last written frame: not writing eval_idx {}".format(
515 self.eval_idx))
516 else:
517 PETSc.Sys.Print("Writing aligned output frame at eval_idx {} (FOM solution)".format(
518 self.eval_idx))
519 # we write the deformation and the solution outputs to the same time step indices
520 if mesh_node_motions is not None:
521 self.mesh.write_deformation_output(mesh_node_motions, self.eval_idx)
522 self.sim_model.write_solution_output(self.eval_idx,
523 solution_array=solution_output)
524 self._last_written_design = design
526 # Explicitly collect garbage and cleanup PETSc objects
527 gc.collect()
528 PETSc.garbage_cleanup(self.mesh.mesh.comm)
530 self._log_memory("after Euler PDE solve")
532 self.eval_idx += 1
534 @staticmethod
535 def _parameter_vector(shape_parameters, alpha):
536 """The ROM snapshot parameter vector of an evaluation: the flat shape
537 parameters (if any), then alpha."""
538 parts = [] if shape_parameters is None else [np.asarray(shape_parameters, dtype=float).ravel()]
539 return np.concatenate(parts + [np.array([float(alpha)])])
541 def _solve(self, parameter_vector):
542 """
543 The accepted flow solution of one evaluation: (owned solution entries,
544 converged).
546 Without a reduced-order model this is the full-order (FOM) solve from
547 the warm start. With one, the ROM is tried first (once it holds enough
548 snapshots), from the warm start projected onto the basis, and its
549 solution is accepted when eta = ||R||/||u|| < eta_threshold; the ROM
550 then holds the adjoint of that solution (rom.adjoint), which
551 apply_inverse_jacobian uses. Otherwise the FOM solves from the warm
552 start exactly as without a ROM, and its solution, if converged and
553 finite, becomes a snapshot at `parameter_vector`.
555 A re-evaluation of the design the FOM solved last skips the ROM: the
556 FOM restarts from its own solution (no Newton steps if that
557 converged), so the design keeps the FOM solution and the full-order
558 adjoint. CSDL re-runs the model before every derivative evaluation,
559 and by then the ROM would reproduce that solution, now a snapshot, and
560 accept it.
561 """
562 # the forward solve needs the memory of the last design's adjoint operators
563 self._release_adjoint_cache()
564 rom = self.reduced_order_model
565 if rom is None:
566 return self._run_fom_solve()
567 rom.discard_adjoint()
569 repeat = self._fom_design is not None and np.array_equal(parameter_vector, self._fom_design[0])
570 rom_result = rom_forces = None
571 if repeat:
572 PETSc.Sys.Print("Re-evaluating the design the FOM solved last: skipping the ROM")
573 else:
574 self.sim_model.interpolate_solution_vector(self.previous_solution)
575 rom_result = rom.solve(self.sim_model, parameter_vector)
576 if rom_result is not None:
577 rom_forces = self._log_rom_solve(rom_result)
578 accept_rom = rom_result is not None and rom_result.accepted
579 if accept_rom and not rom.run_fom_every_evaluation:
580 PETSc.Sys.Print("Accepting ROM solution")
581 self._fom_design = None
582 return rom_result.solution, True
584 solution_fom, fom_converged = self._run_fom_solve()
585 if rom_forces is not None and fom_converged:
586 self._log_rom_error(rom_forces)
587 self._log_rom_state_error(rom_result.solution, solution_fom)
588 if accept_rom:
589 PETSc.Sys.Print("Accepting ROM solution (FOM run for comparison)")
590 self._fom_design = None
591 return rom_result.solution, True
593 PETSc.Sys.Print("Accepting FOM solution")
594 finite = self.mesh.mesh.comm.allreduce(bool(np.all(np.isfinite(solution_fom))), op=MPI.LAND)
595 converged = fom_converged and finite
596 if converged and repeat and self._fom_design[1]:
597 PETSc.Sys.Print("The snapshot of this design is already stored")
598 elif converged:
599 self.sim_model.set_u_vec(solution_fom)
600 rom.add_snapshot(self.sim_model, parameter_vector)
601 sigma = rom.snapshot_singular_values()
602 if self.mesh.mesh.comm.Get_rank() == 0 and sigma is not None:
603 self.data_store.global_snapshot_sing_vals_per_iteration += [(self.eval_idx, sigma)]
604 else:
605 PETSc.Sys.Print("Not adding a ROM snapshot: the FOM solve did not converge")
606 self._fom_design = (np.array(parameter_vector, dtype=float), converged)
607 return solution_fom, fom_converged
609 def _log_rom_solve(self, result):
610 # coefficients of the ROM solution, logged like the FOM's
611 outputs, (cd, cl, *cms) = self._evaluate_outputs(result.solution)
612 cms = tuple(cms)
613 if self.mesh.mesh.comm.Get_rank() == 0:
614 self.data_store.record_outputs("ROM", self.eval_idx, outputs)
615 self.data_store.ROM_walltime += [(self.eval_idx, result.walltime)]
616 self.data_store.rom_relative_residuals += [(self.eval_idx, result.eta)]
617 self.data_store.ROM_force_coefficients += [(self.eval_idx, cd, cl) + cms]
618 self.data_store.ROM_meets_threshold += [(self.eval_idx, result.accepted)]
619 self.data_store.ROM_lspg_converged += [(self.eval_idx, result.converged)]
620 PETSc.Sys.Print("ROM coefficients: c_d: {}, c_l: {}".format(cd, cl) + self._moment_log_string(cms))
621 return (cd, cl) + cms
623 def _log_rom_error(self, rom_forces):
624 # ROM-vs-FOM coefficient errors against the FOM solve just run
625 errors = tuple(float(r - f) for r, f in zip(rom_forces, self._last_fom_coefficients))
626 if self.mesh.mesh.comm.Get_rank() == 0:
627 self.data_store.rom_coefficient_errors += [(self.eval_idx,) + errors]
628 PETSc.Sys.Print("ROM - FOM coefficient errors: c_d: {:.3e}, c_l: {:.3e}".format(*errors[:2])
629 + "".join(", c_m_{}: {:.3e}".format(spec.name, e)
630 for spec, e in zip(self.postprocessor.moment_specs, errors[2:])))
632 def _evaluate_outputs(self, solution):
633 """Every output of the postprocessor for a flow state: ({name: value},
634 (c_d, c_l, c_m_1, ...)). The names are DG_postprocessor.evaluate's (L, D,
635 c_l, c_d, M_<name>, c_m_<name>); the tuple is the layout of the
636 *_force_coefficients records, one moment per moment spec, in order."""
637 L, c_l = self.postprocessor.compute_cl(solution)
638 D, c_d = self.postprocessor.compute_cd(solution)
639 outputs = {"L": L, "D": D, "c_l": c_l, "c_d": c_d}
640 cms = []
641 for spec in self.postprocessor.moment_specs:
642 M, c_m = self.postprocessor.compute_moment(solution, spec)
643 outputs["M_" + spec.name], outputs["c_m_" + spec.name] = M, c_m
644 cms.append(c_m)
645 return outputs, (c_d, c_l, *cms)
647 def _log_rom_state_error(self, solution_rom, solution_fom):
648 # absolute and relative L2 errors of the ROM state against the FOM state just
649 # run, over the domain and over the wall. Both are this rank's owned entries.
650 if not self.mesh.mesh.comm.allreduce(bool(np.all(np.isfinite(solution_rom))), op=MPI.LAND):
651 return
652 diff = np.subtract(solution_fom, solution_rom)
653 l2, l2_rel = self.sim_model.compute_L2_norm(diff, normalization_vec=solution_fom)
654 l2_wall, l2_wall_rel = self.sim_model.compute_L2_norm(
655 diff, normalization_vec=solution_fom, wall_tag=self.WALL_TAG)
656 if self.mesh.mesh.comm.Get_rank() == 0:
657 ds = self.data_store
658 ds.absolute_l2_errors_per_iteration += [(self.eval_idx, l2)]
659 ds.relative_l2_errors_per_iteration += [(self.eval_idx, l2_rel)]
660 ds.absolute_l2_bdry_errors_per_iteration += [(self.eval_idx, l2_wall)]
661 ds.relative_l2_bdry_errors_per_iteration += [(self.eval_idx, l2_wall_rel)]
662 PETSc.Sys.Print("ROM L2 error relative to the FOM: domain {:.3e}, wall {:.3e}".format(l2_rel, l2_wall_rel))
664 def _moment_log_string(self, coefficients):
665 return "".join(", c_m_{}: {}".format(spec.name, c)
666 for spec, c in zip(self.postprocessor.moment_specs, coefficients))
668 def _run_fom_solve(self):
669 # Run the FOM model and output its solution
671 # store pre-FOM time stamp for performance profiling
672 fom_pre_time = perf_counter()
674 # use the most recent converged solution as warm start
675 initial_condition = self.previous_solution
676 self.sim_model.interpolate_solution_vector(initial_condition)
678 # solve the FOM
679 # an incoming state that already meets the convergence criterion
680 # needs no solver (SNES, KSP/PC, matrices): skip building it
681 if self.sim_model.incoming_state_converged():
682 residual_norm = self.sim_model.residual_vec_physical.norm()
683 else:
684 self.sim_model.set_up_solver()
685 residual_norm = self.sim_model.solve_system().norm()
686 fom_converged = self.sim_model.solver_converged
687 eta = self.sim_model.relative_residual_norm(residual_norm)
688 solution_fom = self.sim_model.u_vec.x.petsc_vec.getArray().copy()
689 if self.release_solver_memory:
690 self.sim_model.release_solver()
692 self._log_memory("after FOM solve, before doing anything else")
694 # store post-FOM time stamp for performance profiling
695 fom_post_time = perf_counter()
696 PETSc.Sys.Print("FOM wall time: {}".format(fom_post_time - fom_pre_time))
698 outputs_fom, (cd_fom, cl_fom, *cms_fom) = self._evaluate_outputs(solution_fom)
699 cms_fom = tuple(cms_fom)
700 self._last_fom_coefficients = (cd_fom, cl_fom) + cms_fom
702 if self.mesh.mesh.comm.Get_rank() == 0:
703 self.data_store.record_outputs("FOM", self.eval_idx, outputs_fom)
704 self.data_store.FOM_walltime += [(self.eval_idx, fom_post_time - fom_pre_time)]
705 self.data_store.fom_relative_residuals += [(self.eval_idx, eta)]
706 self.data_store.FOM_force_coefficients += [(self.eval_idx, cd_fom, cl_fom) + cms_fom]
707 PETSc.Sys.Print("FOM coefficients: c_d: {}, c_l: {}".format(cd_fom, cl_fom) + self._moment_log_string(cms_fom))
708 PETSc.Sys.Print("FOM relative residual norm eta: {} (||r||: {})".format(
709 eta, residual_norm))
711 return solution_fom, fom_converged
713 def apply_inverse_jacobian(self, input_vals, outputs, d_outputs, d_residuals, mode):
714 pre_apply_inverse_jacobian_time = perf_counter()
715 # Here we solve the adjoint equation when calculating derivatives for optimization
716 print("starting apply_inverse_jacobian in DG_windtunnel_model...")
718 mesh_node_motions = input_vals['mesh_node_motions'] if 'mesh_node_motions' in input_vals else None
719 shape_parameters = input_vals['shape_parameters'] if 'shape_parameters' in input_vals else None
720 alpha = input_vals['alpha'][0]
721 u_vec = outputs['u_vec'] # numpy array
723 if mode == 'rev':
724 # compute d_residual = (dr_du^-1)*d_outputs
726 # apply the input mesh node motions to the mesh of the sim model
727 # and set the angle of attack before constructing the weak form
728 # and the corresponding solution that we are linearizing around
729 if mesh_node_motions is not None:
730 PETSc.Sys.Print("apply_inverse_jacobian, stepping into mesh.apply_node_motions")
731 self.mesh.apply_node_motions(mesh_node_motions, design=shape_parameters)
732 self.sim_model.set_angle_of_attack(alpha)
733 self.sim_model.compute_weakform()
734 self.sim_model.set_u_vec(u_vec)
736 rom = self.reduced_order_model
737 if (rom is not None and rom.adjoint is not None
738 and rom.adjoint.matches(self._parameter_vector(shape_parameters, alpha), u_vec)):
739 # The accepted solution is the ROM's: the adjoint of its
740 # projected (LSPG) equations, lifted to the full state, which
741 # compute_jacvec_product then uses like the full-order one
742 d_residuals['u_vec'] = rom.adjoint.solve(self.sim_model, d_outputs['u_vec'])
743 self._log_adjoint("rom", pre_apply_inverse_jacobian_time)
744 if self.compute_all_adjoint_errors:
745 # after the ROM adjoint's own timing; d_residuals is not touched
746 self._log_adjoint_error(input_vals, u_vec, alpha, mesh_node_motions,
747 d_outputs, d_residuals['u_vec'])
748 return
750 self._fom_adjoint(input_vals, u_vec, alpha, mesh_node_motions, d_outputs, d_residuals,
751 pre_apply_inverse_jacobian_time)
752 else:
753 raise ValueError("Reverse mode required")
755 def _fom_adjoint(self, input_vals, u_vec, alpha, mesh_node_motions, d_outputs, d_residuals,
756 start_time, log=True):
757 """The full-order adjoint psi = (dR/dU)^-T d_outputs['u_vec'] at the model's current
758 state, written to d_residuals['u_vec']. log=False leaves it out of data_store.adjoint_solves
759 (the reference solve of compute_all_adjoint_errors)."""
760 if self.sim_model.has_condensed_jacobian:
761 self._condensed_adjoint(d_outputs, d_residuals,
762 self._adjoint_state_key(u_vec, alpha, mesh_node_motions))
763 if self.release_solver_memory:
764 self.sim_model.release_solver()
765 PETSc.garbage_cleanup()
766 if log:
767 self._log_adjoint("fom", start_time)
768 return
770 self._drdu_mat = self.sim_model.assemble_dRdu_mat()
771 if not hasattr(self, '_d_output_petsc'):
772 self._d_output_petsc = self._drdu_mat.createVecLeft()
773 self._d_residual_petsc = self._drdu_mat.createVecLeft()
774 self._F_form = dolfinx.fem.form(self.sim_model.F)
775 self._residual_vec_petsc = dolfinx.fem.petsc.create_vector(
776 dolfinx.fem.extract_function_spaces(self._F_form))
778 with self._residual_vec_petsc.localForm() as loc:
779 loc.set(0.0)
780 dolfinx.fem.petsc.assemble_vector(self._residual_vec_petsc, self._F_form)
781 self._residual_vec_petsc.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
782 PETSc.Sys.Print("max residual entry: {}".format(self._residual_vec_petsc.max()[1]))
783 residual_norm = self._residual_vec_petsc.norm()
784 PETSc.Sys.Print("forward eval residual norm: {}".format(residual_norm))
786 # (no assembly here any more: assemble_dRdu_mat above already
787 # returns the matrix assembled. It hands back the same cached
788 # Mat object every call, so the KSP's operator reference below
789 # stays valid.)
790 if not hasattr(self, '_adj_solver'):
791 self._adj_solver = PETSc.KSP().create(self.mesh.mesh.comm)
792 self._adj_solver.setOperators(self._drdu_mat)
793 self._adj_solver.setOptionsPrefix("adj_")
795 # define the adjoint solver settings
796 self.sim_model.apply_krylov_solver_settings(
797 self._adj_solver, max_it=self.linear_solver_max_it,
798 gmres_restart=200,
799 monitor_convergence=False,
800 rtol=self.linear_solver_rtol,
801 atol=self.linear_solver_atol)
803 set_petsc_vec_array(self._d_output_petsc, d_outputs['u_vec'])
805 # solve the transposed linear system and output the solution
806 self._adj_solver.solveTranspose(self._d_output_petsc, self._d_residual_petsc)
808 rnorm = self._adj_solver.getResidualNorm()
809 PETSc.Sys.Print("adj_ solve: {} iterations, residual norm {} [{}]".format(
810 self._adj_solver.getIterationNumber(), rnorm,
811 ksp_converged_reason_name(self._adj_solver.getConvergedReason())))
813 if rnorm > self.linear_residual_accept_limit:
814 raise ValueError(
815 "apply_inverse_jacobian residual norm is too high: {:.6e} > "
816 "linear_residual_accept_limit {:.1e}".format(
817 rnorm, self.linear_residual_accept_limit))
819 PETSc.Sys.Print("d_residual norm: {}".format(self._d_residual_petsc.norm()))
821 if mesh_node_motions is not None:
822 PETSc.Sys.Print("mesh_node_motions min, max: {}, {}".format(mesh_node_motions.min(), mesh_node_motions.max()))
823 PETSc.Sys.Print("shape parameters: {}".format(input_vals['shape_parameters']))
824 PETSc.Sys.Print("alpha: {}".format(np.degrees(alpha)))
826 d_residuals['u_vec'] = self._d_residual_petsc.getArray()
828 PETSc.garbage_cleanup()
830 if log:
831 self._log_adjoint("fom", start_time)
833 def _log_adjoint_error(self, input_vals, u_vec, alpha, mesh_node_motions, d_outputs, psi_rom):
834 """Record the error of the ROM adjoint psi_rom against the full-order adjoint at the
835 same state, design and right-hand side: (eval_idx, output row, absolute 2-norm error,
836 relative error, ||psi_fom||) in data_store.adjoint_errors. The row is the order of the
837 adjoint solves within the evaluation, i.e. of CSDL's derivative rows. Collective."""
838 scratch = {}
839 try:
840 self._fom_adjoint(input_vals, u_vec, alpha, mesh_node_motions, d_outputs, scratch,
841 perf_counter(), log=False)
842 except ValueError as exc:
843 # a diagnostic must not stop the optimization
844 PETSc.Sys.Print("adjoint error: the full-order reference adjoint failed: {}".format(exc))
845 return
846 # copy: the plain path returns a view of a PETSc vector reused by the next solve
847 psi_fom = np.array(scratch['u_vec'], dtype=float)
848 comm = self.mesh.mesh.comm
849 diff = comm.allreduce(float(np.sum((np.asarray(psi_rom) - psi_fom)**2)), op=MPI.SUM)
850 ref = comm.allreduce(float(psi_fom @ psi_fom), op=MPI.SUM)
851 abs_err, fom_norm = np.sqrt(diff), np.sqrt(ref)
852 rel_err = abs_err / fom_norm if fom_norm > 0.0 else np.nan
854 eval_idx = self.eval_idx - 1
855 row = self._adjoint_row[1] if self._adjoint_row[0] == eval_idx else 0
856 self._adjoint_row = (eval_idx, row + 1)
857 if comm.Get_rank() == 0:
858 self.data_store.adjoint_errors += [(eval_idx, row, float(abs_err), float(rel_err), float(fom_norm))]
859 PETSc.Sys.Print("ROM adjoint error (output row {}): |psi_rom - psi_fom| = {:.3e}, relative {:.3e}".format(
860 row, abs_err, rel_err))
862 def _log_adjoint(self, kind, start_time):
863 # which adjoint ('rom' or 'fom') the derivative of the last evaluation
864 # used, and its wall time
865 walltime = perf_counter() - start_time
866 if self.mesh.mesh.comm.Get_rank() == 0:
867 self.data_store.adjoint_solves += [(self.eval_idx - 1, kind, walltime)]
868 PETSc.Sys.Print("apply_inverse_jacobian total time ({} adjoint): {}".format(kind, walltime))
870 def _condensed_adjoint(self, d_outputs, d_residuals, state_key):
871 """The adjoint solve for a model whose dR/dU is a condensed product
872 (a reconstructed gradient): the model's own transposed linearized
873 solve (CompressibleEulerModel.linearized_solver). The transposed
874 matrix and the preconditioned KSP are built once per state and reused
875 by the adjoint solves of all outputs at it."""
876 if self._adjoint_cache_matches(state_key):
877 PETSc.Sys.Print("adj_: reusing the transposed Jacobian and preconditioner of this state")
878 else:
879 self._release_adjoint_cache()
880 if self.release_solver_memory:
881 # free queued PETSc objects before the large assembly
882 gc.collect()
883 PETSc.garbage_cleanup(self.mesh.mesh.comm)
884 # the condensed dR/dU is freed once its transposes exist: only they are used from then on
885 ksp = self.sim_model.linearized_solver(
886 transpose=True, prefix="adj_", rtol=self.linear_solver_rtol,
887 atol=self.linear_solver_atol, max_it=self.linear_solver_max_it,
888 release_operands=True, matrix_free=self.adjoint_matrix_free,
889 ksp_options=self.adjoint_ksp_options)
890 self._adjoint_cache = {'key': state_key, 'ksp': ksp}
891 ksp = self._adjoint_cache['ksp']
892 A = ksp.getOperators()[0]
893 rhs, sol = A.createVecLeft(), A.createVecLeft()
894 set_petsc_vec_array(rhs, d_outputs['u_vec'])
895 guess_used = self.adjoint_first_order_guess and self._first_order_adjoint_guess(ksp, A, rhs, sol)
896 ksp.setInitialGuessNonzero(bool(guess_used))
897 ksp.solve(rhs, sol)
898 rnorm = ksp.getResidualNorm()
899 PETSc.Sys.Print("adj_ solve: {} iterations, residual norm {} [{}]".format(
900 ksp.getIterationNumber(), rnorm, ksp_converged_reason_name(ksp.getConvergedReason())))
901 if rnorm > self.linear_residual_accept_limit:
902 raise ValueError(
903 "apply_inverse_jacobian residual norm is too high: {:.6e} > "
904 "linear_residual_accept_limit {:.1e}".format(rnorm, self.linear_residual_accept_limit))
905 d_residuals['u_vec'] = sol.getArray().copy()
906 rhs.destroy()
907 sol.destroy()
909 # ---- adjoint reuse: cached operators -----------------------------------
910 def _first_order_adjoint_guess(self, ksp, A, rhs, sol):
911 """
912 Fill `sol` with the first-order adjoint solution of `rhs`: the solution
913 of P x = rhs, P the transposed compact part A_UU of the Jacobian (the
914 KSP's preconditioning matrix), to a relative tolerance
915 first_order_adjoint_rtol. The KSP is switched to operator P for the
916 solve, which keeps its preconditioner (P's factors), and back to
917 (A, P) afterwards, with its tolerances. Returns False, leaving `sol`
918 zero, if the first-order solve did not converge. The residual of the
919 exact system at the guess, |rhs - A guess| / |rhs|, the iterations and
920 the time are logged and kept in self._last_first_order_guess.
921 Collective.
922 """
923 P = ksp.getOperators()[1]
924 if P.handle == A.handle:
925 return False # direct solver: A is P, nothing to precondition with
926 pre_time = perf_counter()
927 rtol, atol, divtol, max_it = ksp.getTolerances()
928 ksp.setOperators(P, P)
929 ksp.setTolerances(rtol=self.first_order_adjoint_rtol, atol=0.0, max_it=self.linear_solver_max_it)
930 ksp.setInitialGuessNonzero(False)
931 try:
932 ksp.solve(rhs, sol)
933 iterations, reason = ksp.getIterationNumber(), ksp.getConvergedReason()
934 finally:
935 ksp.setOperators(A, P)
936 ksp.setTolerances(rtol=rtol, atol=atol, divtol=divtol, max_it=max_it)
937 if reason <= 0:
938 PETSc.Sys.Print("adj_ first-order guess: no convergence after {} iterations [{}]: zero guess".format(
939 iterations, ksp_converged_reason_name(reason)))
940 sol.zeroEntries()
941 self._last_first_order_guess = dict(iterations=iterations, ratio=np.nan, used=False)
942 return False
943 residual = rhs.duplicate()
944 A.mult(sol, residual)
945 residual.aypx(-1.0, rhs)
946 ratio = residual.norm() / rhs.norm()
947 residual.destroy()
948 walltime = perf_counter() - pre_time
949 PETSc.Sys.Print("adj_ first-order guess: {} iterations, {:.1f} s; the exact system's initial residual is "
950 "{:.3e} of |rhs|".format(iterations, walltime, ratio))
951 self._last_first_order_guess = dict(iterations=iterations, ratio=float(ratio), used=True, walltime=walltime)
952 return True
954 @staticmethod
955 def _adjoint_state_key(u_vec, alpha, mesh_node_motions):
956 """What the adjoint's matrix depends on: the state, alpha and the mesh motion."""
957 return (np.array(u_vec, copy=True), float(alpha),
958 None if mesh_node_motions is None else np.array(mesh_node_motions, copy=True))
960 def _state_keys_equal(self, key0, key):
961 u0, alpha0, motions0 = key0
962 u, alpha, motions = key
963 same = (alpha == alpha0 and np.array_equal(u0, u) and (motions0 is None) == (motions is None)
964 and (motions is None or np.array_equal(motions0, motions)))
965 return bool(self.mesh.mesh.comm.allreduce(int(same), op=MPI.MIN))
967 def _adjoint_cache_matches(self, state_key):
968 cache = self._adjoint_cache
969 return cache is not None and self._state_keys_equal(cache['key'], state_key)
971 def _release_adjoint_cache(self):
972 """Free what the adjoint of the last state keeps: the transposed matrices
973 and KSP, the cached jacvec operators and the derived fields' cached
974 Jacobians."""
975 cache, self._adjoint_cache = self._adjoint_cache, None
976 freed = cache is not None
977 if freed:
978 self.sim_model.destroy_adjoint_operator(cache['ksp'])
979 del cache
980 if self._dRdx_cache is not None:
981 destroy_chained_operator(self._dRdx_cache['op'])
982 self._dRdx_cache = None
983 freed = True
984 if self._dRdalpha_cache is not None:
985 self._dRdalpha_cache = None
986 if self.sim_model.derived_fields:
987 self.sim_model.release_derived_caches()
988 freed = True
989 if freed:
990 self.sim_model._return_freed_memory()
992 def set_adjoint_options(self, cell_centre_matrix=False, cache_jacobians=False, structured_gradient=False,
993 matrix_free=False, cache_jacvec_operators=False):
994 """
995 Switch the speed-ups of the adjoint of a model with a condensed Jacobian
996 (all exact reformulations; each off by default). Frees every cache.
997 cell_centre_matrix build the constant cell-centre matrix dc/dx vectorized, once
998 cache_jacobians keep the Green-Gauss Jacobians dG/dU, dG/dalpha, dG/dx
999 while the state, alpha and the mesh are unchanged
1000 structured_gradient apply dG/dU as W (x) I plus a boundary part instead of an
1001 assembled matrix
1002 matrix_free the adjoint operator without forming A or A_UG B
1003 cache_jacvec_operators build the dR/dx operator and dR/dalpha once per design
1004 """
1005 self._release_adjoint_cache()
1006 rec = getattr(self.sim_model, "reconstruction", None)
1007 if rec is not None:
1008 rec.fast_cell_centre_jacobian = bool(cell_centre_matrix)
1009 rec.cache_jacobians = bool(cache_jacobians)
1010 rec.structured_state_jacobian = bool(structured_gradient)
1011 self.adjoint_matrix_free = bool(matrix_free)
1012 self.cache_jacvec_operators = bool(cache_jacvec_operators)
1014 def compute_jacvec_product(self, input_vals, outputs, d_inputs, d_outputs, d_residuals, mode):
1015 # Here we compute the product of the adjoint solution with d_residual
1016 pre_compute_jacvec_product_time = perf_counter()
1017 print("starting compute_jacvec_product in DG_windtunnel_model...")
1019 mesh_node_motions = input_vals['mesh_node_motions'] if 'mesh_node_motions' in input_vals else None
1020 shape_parameters = input_vals['shape_parameters'] if 'shape_parameters' in input_vals else None
1021 alpha = input_vals['alpha'][0]
1022 u_vec = outputs['u_vec'] # numpy array
1024 if mode == 'rev':
1025 # compute d_input = (dr_dinput^T)*d_residual
1027 # apply the input mesh node motions to the mesh of the sim model,
1028 # apply the angle of attack and compute the weak form
1029 if mesh_node_motions is not None:
1030 PETSc.Sys.Print("compute_jacvec_product, stepping into mesh.apply_node_motions")
1031 self.mesh.apply_node_motions(mesh_node_motions, design=shape_parameters)
1032 self.sim_model.set_angle_of_attack(alpha)
1033 self.sim_model.compute_weakform()
1035 if 'alpha' in d_inputs:
1036 # alpha is a single global scalar, so dR/d(alpha) is one column
1037 # and this reverse product is a plain dot product between vectors
1038 local_size = self.sim_model.u_vec.x.petsc_vec.local_size
1039 if self.cache_jacvec_operators and self.sim_model.derived_fields:
1040 key = self._adjoint_state_key(u_vec, alpha, mesh_node_motions)
1041 if self._dRdalpha_cache is None or not self._state_keys_equal(self._dRdalpha_cache[0], key):
1042 dr_dalpha = self.sim_model.assemble_dRdalpha_vec(u_vec)
1043 self._dRdalpha_cache = (key, np.array(dr_dalpha.getArray()[:local_size], copy=True))
1044 dr_dalpha_local = self._dRdalpha_cache[1]
1045 else:
1046 dr_dalpha = self.sim_model.assemble_dRdalpha_vec(u_vec)
1048 # we first have to assert that the MPI objects are compatible
1049 dr_dalpha_local = dr_dalpha.getArray()[:local_size]
1050 assert dr_dalpha_local.shape == d_residuals['u_vec'].shape, (
1051 "dR/dalpha local slice {} does not match d_residuals['u_vec'] {}".format(
1052 dr_dalpha_local.shape, d_residuals['u_vec'].shape))
1054 # we do the multiplication of the local objects through Numpy,
1055 # after which we sum the contributions on every MPI rank
1056 local_contribution = float(np.dot(dr_dalpha_local, d_residuals['u_vec']))
1057 d_inputs['alpha'] = np.array(
1058 [self.mesh.mesh.comm.allreduce(local_contribution, op=MPI.SUM)],
1059 dtype=np.float64)
1061 PETSc.Sys.Print("d_inputs['alpha']: {}".format(d_inputs['alpha']))
1063 if 'mesh_node_motions' in d_inputs:
1064 cached_operator = self.cache_jacvec_operators and bool(self.sim_model.derived_fields)
1065 if cached_operator:
1066 key = self._adjoint_state_key(u_vec, alpha, mesh_node_motions)
1067 if self._dRdx_cache is None or not self._state_keys_equal(self._dRdx_cache['key'], key):
1068 if self._dRdx_cache is not None:
1069 destroy_chained_operator(self._dRdx_cache['op'])
1070 self._dRdx_cache = {'key': key, 'op': self.sim_model.compute_dRdxmesh_mat(
1071 u_vec, self.mesh.coordinate_space)}
1072 dr_dxmesh = self._dRdx_cache['op']
1073 else:
1074 dr_dxmesh = self.sim_model.compute_dRdxmesh_mat(u_vec, self.mesh.coordinate_space)
1076 if not hasattr(self, '_d_residual_jvp_petsc'):
1077 self._d_residual_jvp_petsc = dr_dxmesh.createVecLeft()
1078 self._d_input_jvp_petsc = dr_dxmesh.createVecRight()
1080 set_petsc_vec_array(self._d_residual_jvp_petsc, d_residuals['u_vec'])
1082 # compute d_input
1083 self._d_input_jvp_petsc.set(0.0)
1084 dr_dxmesh.multTranspose(self._d_residual_jvp_petsc, self._d_input_jvp_petsc)
1085 d_input = self._d_input_jvp_petsc
1086 if self.sim_model.derived_fields and not cached_operator:
1087 # a fresh chained operator per call (not cached): free it now
1088 destroy_chained_operator(dr_dxmesh)
1090 PETSc.Sys.Print("d_input norm: {}".format(d_input.norm()))
1092 d_input_cg1 = d_input.getArray().reshape(-1, self.mesh.mesh.geometry.dim) # (N_CG1_dofs, gdim)
1093 d_input_geom = self.mesh.dof_to_node_map.T @ d_input_cg1 # (N_geom_nodes, gdim)
1094 d_inputs['mesh_node_motions'] = d_input_geom
1096 PETSc.garbage_cleanup()
1097 else:
1098 raise ValueError("Reverse mode required")
1100 PETSc.Sys.Print("DG_windtunnel_model compute_jacvec_product done")
1102 post_compute_jacvec_product_time = perf_counter()
1103 total_compute_jacvec_product_time = post_compute_jacvec_product_time - pre_compute_jacvec_product_time
1105 PETSc.Sys.Print("compute_jacvec_product total time: {}".format(total_compute_jacvec_product_time))