Deleted Added
full compact
intel_dmar.h (257251) intel_dmar.h (259512)
1/*-
2 * Copyright (c) 2013 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6 * under sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 12 unchanged lines hidden (view full) ---

21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
1/*-
2 * Copyright (c) 2013 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6 * under sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 12 unchanged lines hidden (view full) ---

21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: head/sys/x86/iommu/intel_dmar.h 257251 2013-10-28 13:33:29Z kib $
29 * $FreeBSD: stable/10/sys/x86/iommu/intel_dmar.h 259512 2013-12-17 13:49:35Z kib $
30 */
31
32#ifndef __X86_IOMMU_INTEL_DMAR_H
33#define __X86_IOMMU_INTEL_DMAR_H
34
35/* Host or physical memory address, after translation. */
36typedef uint64_t dmar_haddr_t;
37/* Guest or bus address, before translation. */
38typedef uint64_t dmar_gaddr_t;
39
30 */
31
32#ifndef __X86_IOMMU_INTEL_DMAR_H
33#define __X86_IOMMU_INTEL_DMAR_H
34
35/* Host or physical memory address, after translation. */
36typedef uint64_t dmar_haddr_t;
37/* Guest or bus address, before translation. */
38typedef uint64_t dmar_gaddr_t;
39
40struct dmar_qi_genseq {
41 u_int gen;
42 uint32_t seq;
43};
44
40struct dmar_map_entry {
41 dmar_gaddr_t start;
42 dmar_gaddr_t end;
43 dmar_gaddr_t free_after; /* Free space after the entry */
44 dmar_gaddr_t free_down; /* Max free space below the
45 current R/B tree node */
46 u_int flags;
47 TAILQ_ENTRY(dmar_map_entry) dmamap_link; /* Link for dmamap entries */
48 RB_ENTRY(dmar_map_entry) rb_entry; /* Links for ctx entries */
49 TAILQ_ENTRY(dmar_map_entry) unroll_link; /* Link for unroll after
50 dmamap_load failure */
45struct dmar_map_entry {
46 dmar_gaddr_t start;
47 dmar_gaddr_t end;
48 dmar_gaddr_t free_after; /* Free space after the entry */
49 dmar_gaddr_t free_down; /* Max free space below the
50 current R/B tree node */
51 u_int flags;
52 TAILQ_ENTRY(dmar_map_entry) dmamap_link; /* Link for dmamap entries */
53 RB_ENTRY(dmar_map_entry) rb_entry; /* Links for ctx entries */
54 TAILQ_ENTRY(dmar_map_entry) unroll_link; /* Link for unroll after
55 dmamap_load failure */
56 struct dmar_ctx *ctx;
57 struct dmar_qi_genseq gseq;
51};
52
53RB_HEAD(dmar_gas_entries_tree, dmar_map_entry);
54RB_PROTOTYPE(dmar_gas_entries_tree, dmar_map_entry, rb_entry,
55 dmar_gas_cmp_entries);
56
57#define DMAR_MAP_ENTRY_PLACE 0x0001 /* Fake entry */
58#define DMAR_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by
59 dmamap_link */
60#define DMAR_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by
61 dmamap_link */
62#define DMAR_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */
58};
59
60RB_HEAD(dmar_gas_entries_tree, dmar_map_entry);
61RB_PROTOTYPE(dmar_gas_entries_tree, dmar_map_entry, rb_entry,
62 dmar_gas_cmp_entries);
63
64#define DMAR_MAP_ENTRY_PLACE 0x0001 /* Fake entry */
65#define DMAR_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by
66 dmamap_link */
67#define DMAR_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by
68 dmamap_link */
69#define DMAR_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */
70#define DMAR_MAP_ENTRY_QI_NF 0x0020 /* qi task, do not free entry */
63#define DMAR_MAP_ENTRY_READ 0x1000 /* Read permitted */
64#define DMAR_MAP_ENTRY_WRITE 0x2000 /* Write permitted */
65#define DMAR_MAP_ENTRY_SNOOP 0x4000 /* Snoop */
66#define DMAR_MAP_ENTRY_TM 0x8000 /* Transient */
67
68struct dmar_ctx {
69 int bus; /* pci bus/slot/func */
70 int slot;

--- 37 unchanged lines hidden (view full) ---

108#define DMAR_CTX_PGUNLOCK(ctx) VM_OBJECT_WUNLOCK((ctx)->pgtbl_obj)
109#define DMAR_CTX_ASSERT_PGLOCKED(ctx) \
110 VM_OBJECT_ASSERT_WLOCKED((ctx)->pgtbl_obj)
111
112#define DMAR_CTX_LOCK(ctx) mtx_lock(&(ctx)->lock)
113#define DMAR_CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->lock)
114#define DMAR_CTX_ASSERT_LOCKED(ctx) mtx_assert(&(ctx)->lock, MA_OWNED)
115
71#define DMAR_MAP_ENTRY_READ 0x1000 /* Read permitted */
72#define DMAR_MAP_ENTRY_WRITE 0x2000 /* Write permitted */
73#define DMAR_MAP_ENTRY_SNOOP 0x4000 /* Snoop */
74#define DMAR_MAP_ENTRY_TM 0x8000 /* Transient */
75
76struct dmar_ctx {
77 int bus; /* pci bus/slot/func */
78 int slot;

--- 37 unchanged lines hidden (view full) ---

116#define DMAR_CTX_PGUNLOCK(ctx) VM_OBJECT_WUNLOCK((ctx)->pgtbl_obj)
117#define DMAR_CTX_ASSERT_PGLOCKED(ctx) \
118 VM_OBJECT_ASSERT_WLOCKED((ctx)->pgtbl_obj)
119
120#define DMAR_CTX_LOCK(ctx) mtx_lock(&(ctx)->lock)
121#define DMAR_CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->lock)
122#define DMAR_CTX_ASSERT_LOCKED(ctx) mtx_assert(&(ctx)->lock, MA_OWNED)
123
124struct dmar_msi_data {
125 int irq;
126 int irq_rid;
127 struct resource *irq_res;
128 void *intr_handle;
129 int (*handler)(void *);
130 int msi_data_reg;
131 int msi_addr_reg;
132 int msi_uaddr_reg;
133 void (*enable_intr)(struct dmar_unit *);
134 void (*disable_intr)(struct dmar_unit *);
135 const char *name;
136};
137
138#define DMAR_INTR_FAULT 0
139#define DMAR_INTR_QI 1
140#define DMAR_INTR_TOTAL 2
141
116struct dmar_unit {
117 device_t dev;
118 int unit;
119 uint16_t segment;
120 uint64_t base;
121
122 /* Resources */
123 int reg_rid;
124 struct resource *regs;
142struct dmar_unit {
143 device_t dev;
144 int unit;
145 uint16_t segment;
146 uint64_t base;
147
148 /* Resources */
149 int reg_rid;
150 struct resource *regs;
125 int irq;
126 int irq_rid;
127 struct resource *irq_res;
128 void *intr_handle;
129
151
152 struct dmar_msi_data intrs[DMAR_INTR_TOTAL];
153
130 /* Hardware registers cache */
131 uint32_t hw_ver;
132 uint64_t hw_cap;
133 uint64_t hw_ecap;
134 uint32_t hw_gcmd;
135
136 /* Data for being a dmar */
137 struct mtx lock;

--- 6 unchanged lines hidden (view full) ---

144 struct mtx fault_lock;
145 uint64_t *fault_log;
146 int fault_log_head;
147 int fault_log_tail;
148 int fault_log_size;
149 struct task fault_task;
150 struct taskqueue *fault_taskqueue;
151
154 /* Hardware registers cache */
155 uint32_t hw_ver;
156 uint64_t hw_cap;
157 uint64_t hw_ecap;
158 uint32_t hw_gcmd;
159
160 /* Data for being a dmar */
161 struct mtx lock;

--- 6 unchanged lines hidden (view full) ---

168 struct mtx fault_lock;
169 uint64_t *fault_log;
170 int fault_log_head;
171 int fault_log_tail;
172 int fault_log_size;
173 struct task fault_task;
174 struct taskqueue *fault_taskqueue;
175
176 /* QI */
177 int qi_enabled;
178 vm_offset_t inv_queue;
179 vm_size_t inv_queue_size;
180 uint32_t inv_queue_avail;
181 uint32_t inv_queue_tail;
182 volatile uint32_t inv_waitd_seq_hw; /* hw writes there on wait
183 descr completion */
184 uint64_t inv_waitd_seq_hw_phys;
185 uint32_t inv_waitd_seq; /* next sequence number to use for wait descr */
186 u_int inv_waitd_gen; /* seq number generation AKA seq overflows */
187 u_int inv_seq_waiters; /* count of waiters for seq */
188 u_int inv_queue_full; /* informational counter */
189
190 /* Delayed freeing of map entries queue processing */
191 struct dmar_map_entries_tailq tlb_flush_entries;
192 struct task qi_task;
193 struct taskqueue *qi_taskqueue;
194
152 /* Busdma delayed map load */
153 struct task dmamap_load_task;
154 TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;
155 struct taskqueue *delayed_taskqueue;
156};
157
158#define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock)
159#define DMAR_UNLOCK(dmar) mtx_unlock(&(dmar)->lock)
160#define DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->lock, MA_OWNED)
161
162#define DMAR_FAULT_LOCK(dmar) mtx_lock_spin(&(dmar)->fault_lock)
163#define DMAR_FAULT_UNLOCK(dmar) mtx_unlock_spin(&(dmar)->fault_lock)
164#define DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED)
165
166#define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
195 /* Busdma delayed map load */
196 struct task dmamap_load_task;
197 TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;
198 struct taskqueue *delayed_taskqueue;
199};
200
201#define DMAR_LOCK(dmar) mtx_lock(&(dmar)->lock)
202#define DMAR_UNLOCK(dmar) mtx_unlock(&(dmar)->lock)
203#define DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->lock, MA_OWNED)
204
205#define DMAR_FAULT_LOCK(dmar) mtx_lock_spin(&(dmar)->fault_lock)
206#define DMAR_FAULT_UNLOCK(dmar) mtx_unlock_spin(&(dmar)->fault_lock)
207#define DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED)
208
209#define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
210#define DMAR_HAS_QI(dmar) (((dmar)->hw_ecap & DMAR_ECAP_QI) != 0)
167
168/* Barrier ids */
169#define DMAR_BARRIER_RMRR 0
170#define DMAR_BARRIER_USEQ 1
171
172struct dmar_unit *dmar_find(device_t dev);
173
174u_int dmar_nd2mask(u_int nd);
175bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);
176int ctx_set_agaw(struct dmar_ctx *ctx, int mgaw);
177int dmar_maxaddr2mgaw(struct dmar_unit* unit, dmar_gaddr_t maxaddr,
178 bool allow_less);
179vm_pindex_t pglvl_max_pages(int pglvl);
180int ctx_is_sp_lvl(struct dmar_ctx *ctx, int lvl);
181dmar_gaddr_t pglvl_page_size(int total_pglvl, int lvl);
182dmar_gaddr_t ctx_page_size(struct dmar_ctx *ctx, int lvl);
211
212/* Barrier ids */
213#define DMAR_BARRIER_RMRR 0
214#define DMAR_BARRIER_USEQ 1
215
216struct dmar_unit *dmar_find(device_t dev);
217
218u_int dmar_nd2mask(u_int nd);
219bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);
220int ctx_set_agaw(struct dmar_ctx *ctx, int mgaw);
221int dmar_maxaddr2mgaw(struct dmar_unit* unit, dmar_gaddr_t maxaddr,
222 bool allow_less);
223vm_pindex_t pglvl_max_pages(int pglvl);
224int ctx_is_sp_lvl(struct dmar_ctx *ctx, int lvl);
225dmar_gaddr_t pglvl_page_size(int total_pglvl, int lvl);
226dmar_gaddr_t ctx_page_size(struct dmar_ctx *ctx, int lvl);
227int calc_am(struct dmar_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size,
228 dmar_gaddr_t *isizep);
183struct vm_page *dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags);
184void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags);
185void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags,
186 struct sf_buf **sf);
187void dmar_unmap_pgtbl(struct sf_buf *sf, bool coherent);
188int dmar_load_root_entry_ptr(struct dmar_unit *unit);
189int dmar_inv_ctx_glob(struct dmar_unit *unit);
190int dmar_inv_iotlb_glob(struct dmar_unit *unit);
191int dmar_flush_write_bufs(struct dmar_unit *unit);
192int dmar_enable_translation(struct dmar_unit *unit);
193int dmar_disable_translation(struct dmar_unit *unit);
229struct vm_page *dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags);
230void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags);
231void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags,
232 struct sf_buf **sf);
233void dmar_unmap_pgtbl(struct sf_buf *sf, bool coherent);
234int dmar_load_root_entry_ptr(struct dmar_unit *unit);
235int dmar_inv_ctx_glob(struct dmar_unit *unit);
236int dmar_inv_iotlb_glob(struct dmar_unit *unit);
237int dmar_flush_write_bufs(struct dmar_unit *unit);
238int dmar_enable_translation(struct dmar_unit *unit);
239int dmar_disable_translation(struct dmar_unit *unit);
194void dmar_enable_intr(struct dmar_unit *unit);
195void dmar_disable_intr(struct dmar_unit *unit);
196bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
197void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);
198
240bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
241void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);
242
199int dmar_intr(void *arg);
243int dmar_fault_intr(void *arg);
244void dmar_enable_fault_intr(struct dmar_unit *unit);
245void dmar_disable_fault_intr(struct dmar_unit *unit);
200int dmar_init_fault_log(struct dmar_unit *unit);
201void dmar_fini_fault_log(struct dmar_unit *unit);
202
246int dmar_init_fault_log(struct dmar_unit *unit);
247void dmar_fini_fault_log(struct dmar_unit *unit);
248
249int dmar_qi_intr(void *arg);
250void dmar_enable_qi_intr(struct dmar_unit *unit);
251void dmar_disable_qi_intr(struct dmar_unit *unit);
252int dmar_init_qi(struct dmar_unit *unit);
253void dmar_fini_qi(struct dmar_unit *unit);
254void dmar_qi_invalidate_locked(struct dmar_ctx *ctx, dmar_gaddr_t start,
255 dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
256void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
257void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
258
203vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);
204void put_idmap_pgtbl(vm_object_t obj);
205int ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
206 vm_page_t *ma, uint64_t pflags, int flags);
207int ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
208 int flags);
259vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);
260void put_idmap_pgtbl(vm_object_t obj);
261int ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
262 vm_page_t *ma, uint64_t pflags, int flags);
263int ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
264 int flags);
265void ctx_flush_iotlb_sync(struct dmar_ctx *ctx, dmar_gaddr_t base,
266 dmar_gaddr_t size);
209int ctx_alloc_pgtbl(struct dmar_ctx *ctx);
210void ctx_free_pgtbl(struct dmar_ctx *ctx);
211
212struct dmar_ctx *dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev,
213 bool rmrr);
214struct dmar_ctx *dmar_get_ctx(struct dmar_unit *dmar, device_t dev,
215 bool id_mapped, bool rmrr_init);
216void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
217void dmar_free_ctx(struct dmar_ctx *ctx);
218struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, int bus,
219 int slot, int func);
267int ctx_alloc_pgtbl(struct dmar_ctx *ctx);
268void ctx_free_pgtbl(struct dmar_ctx *ctx);
269
270struct dmar_ctx *dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev,
271 bool rmrr);
272struct dmar_ctx *dmar_get_ctx(struct dmar_unit *dmar, device_t dev,
273 bool id_mapped, bool rmrr_init);
274void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
275void dmar_free_ctx(struct dmar_ctx *ctx);
276struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, int bus,
277 int slot, int func);
278void dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free);
220void dmar_ctx_unload(struct dmar_ctx *ctx,
221 struct dmar_map_entries_tailq *entries, bool cansleep);
279void dmar_ctx_unload(struct dmar_ctx *ctx,
280 struct dmar_map_entries_tailq *entries, bool cansleep);
281void dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free);
222
223int dmar_init_busdma(struct dmar_unit *unit);
224void dmar_fini_busdma(struct dmar_unit *unit);
225
226void dmar_gas_init_ctx(struct dmar_ctx *ctx);
227void dmar_gas_fini_ctx(struct dmar_ctx *ctx);
228struct dmar_map_entry *dmar_gas_alloc_entry(struct dmar_ctx *ctx, u_int flags);
229void dmar_gas_free_entry(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
230void dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
231int dmar_gas_map(struct dmar_ctx *ctx, const struct bus_dma_tag_common *common,
232 dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,
233 struct dmar_map_entry **res);
282
283int dmar_init_busdma(struct dmar_unit *unit);
284void dmar_fini_busdma(struct dmar_unit *unit);
285
286void dmar_gas_init_ctx(struct dmar_ctx *ctx);
287void dmar_gas_fini_ctx(struct dmar_ctx *ctx);
288struct dmar_map_entry *dmar_gas_alloc_entry(struct dmar_ctx *ctx, u_int flags);
289void dmar_gas_free_entry(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
290void dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
291int dmar_gas_map(struct dmar_ctx *ctx, const struct bus_dma_tag_common *common,
292 dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,
293 struct dmar_map_entry **res);
294void dmar_gas_free_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
234int dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
235 u_int eflags, u_int flags, vm_page_t *ma);
236int dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,
237 dmar_gaddr_t end);
238
239void dmar_ctx_parse_rmrr(struct dmar_ctx *ctx, device_t dev,
240 struct dmar_map_entries_tailq *rmrr_entries);
241int dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar);

--- 133 unchanged lines hidden ---
295int dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
296 u_int eflags, u_int flags, vm_page_t *ma);
297int dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,
298 dmar_gaddr_t end);
299
300void dmar_ctx_parse_rmrr(struct dmar_ctx *ctx, device_t dev,
301 struct dmar_map_entries_tailq *rmrr_entries);
302int dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar);

--- 133 unchanged lines hidden ---